Merge release branch 4.20 to main

* 4.20:
  UI: Fix userdata and load balancer selection (#10016)
  Prevent password updates for SAML and LDAP users (#9999)
  cloudstack-migrate-databases: sql AND added (#10033)
  engine/schema: move SQLs to 4.20.0 to 4.20.1 upgrade (#10018)
  Remove user from project before deletion (#10008)
  Simplify validation for creating volume templates via UI (#9828)

Author: DaanHoogland

api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java

@@ -42,7 +42,7 @@ public class ListMgmtsCmd extends BaseListCmd {
 
     @Parameter(name = ApiConstants.PEERS, type = CommandType.BOOLEAN,
             description = "Whether to return the management server peers or not. By default, the management server peers will not be returned.",
-            since = "4.20.0.0")
+            since = "4.20.1.0")
     private Boolean peers;
 
     /////////////////////////////////////////////////////

engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java

@@ -47,6 +47,8 @@ public interface ProjectAccountDao extends GenericDao<ProjectAccountVO, Long> {
 
     void removeAccountFromProjects(long accountId);
 
+    void removeUserFromProjects(long userId);
+
     boolean canUserModifyProject(long projectId, long accountId, long userId);
 
     List<ProjectAccountVO> listUsersOrAccountsByRole(long id);

engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java

@@ -192,6 +192,17 @@ public void removeAccountFromProjects(long accountId) {
         }
     }
 
+    @Override
+    public void removeUserFromProjects(long userId) {
+        SearchCriteria<ProjectAccountVO> sc = AllFieldsSearch.create();
+        sc.setParameters("userId", userId);
+
+        int removedCount = remove(sc);
+        if (removedCount > 0) {
+            logger.debug(String.format("Removed user [%s] from %s project(s).", userId, removedCount));
+        }
+    }
+
     @Override
     public boolean canUserModifyProject(long projectId, long accountId, long userId) {
         SearchCriteria<ProjectAccountVO> sc = AllFieldsSearch.create();

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+public class Upgrade41910to41920 implements DbUpgrade {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.19.1.0", "4.19.2.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.19.2.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-41910to41920.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-41910to41920-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+}

engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql

@@ -0,0 +1,23 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.19.1.0 to 4.19.2.0
+--;
+
+-- Delete `project_account` entries for users that were removed
+DELETE FROM `cloud`.`project_account` WHERE `user_id` IN (SELECT `id` FROM `cloud`.`user` WHERE `removed`);

engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql

@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.19.1.0 to 4.19.2.0
+--;

engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql

@@ -425,10 +425,3 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervi
 
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for vm" ');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for volumes" ');
-
--- Modify index for mshost_peer
-DELETE FROM `cloud`.`mshost_peer`;
-CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost');
-CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
-CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
-CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -22,3 +22,10 @@
 -- Add column api_key_access to user and account tables
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the user" AFTER `secret_key`');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" ');
+
+-- Modify index for mshost_peer
+DELETE FROM `cloud`.`mshost_peer`;
+CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost');
+CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
+CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
+CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');

framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java

@@ -656,7 +656,7 @@ private void migrateTemplateDeployAsIsDetails(Connection conn) throws SQLExcepti
         String sqlTemplateDeployAsIsDetails = "SELECT template_deploy_as_is_details.value " +
                 "FROM template_deploy_as_is_details JOIN vm_instance " +
                 "WHERE template_deploy_as_is_details.template_id = vm_instance.vm_template_id " +
-                "vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1";
+                "AND vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1";
         try (PreparedStatement selectPstmt = conn.prepareStatement("SELECT id, vm_id, name, value FROM user_vm_deploy_as_is_details");
              ResultSet rs = selectPstmt.executeQuery();
              PreparedStatement updatePstmt = conn.prepareStatement("UPDATE user_vm_deploy_as_is_details SET value=? WHERE id=?")

server/src/main/java/com/cloud/user/AccountManagerImpl.java

@@ -1500,6 +1500,8 @@ public UserAccount updateUser(UpdateUserCmd updateUserCmd) {
      * <ul>
      *  <li> If 'password' is blank, we throw an {@link InvalidParameterValueException};
      *  <li> If 'current password' is not provided and user is not an Admin, we throw an {@link InvalidParameterValueException};
+     *  <li> If the user whose password is being changed has a source equal to {@link User.Source#SAML2}, {@link User.Source#SAML2DISABLED} or {@link User.Source#LDAP},
+     *      we throw an {@link InvalidParameterValueException};
      *  <li> If a normal user is calling this method, we use {@link #validateCurrentPassword(UserVO, String)} to check if the provided old password matches the database one;
      * </ul>
      *
@@ -1514,6 +1516,12 @@ public void validateUserPasswordAndUpdateIfNeeded(String newPassword, UserVO use
             throw new InvalidParameterValueException("Password cannot be empty or blank.");
         }
 
+        User.Source userSource = user.getSource();
+        if (userSource == User.Source.SAML2 || userSource == User.Source.SAML2DISABLED || userSource == User.Source.LDAP) {
+            logger.warn(String.format("Unable to update the password for user [%d], as its source is [%s].", user.getId(), user.getSource().toString()));
+            throw new InvalidParameterValueException("CloudStack does not support updating passwords for SAML or LDAP users. Please contact your cloud administrator for assistance.");
+        }
+
         passwordPolicy.verifyIfPasswordCompliesWithPasswordPolicies(newPassword, user.getUsername(), getAccount(user.getAccountId()).getDomainId());
 
         Account callingAccount = getCurrentCallingAccount();