Merge branch '4.19'

Author: DaanHoogland

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -130,6 +130,7 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
     protected static Logger LOGGER = LogManager.getLogger(DatabaseUpgradeChecker.class);
     private final DatabaseVersionHierarchy hierarchy;
     private static final String VIEWS_DIRECTORY = Paths.get("META-INF", "db", "views").toString();
+    private static final String PROCEDURES_DIRECTORY = Paths.get("META-INF", "db", "procedures").toString();
 
     @Inject
     VersionDao _dao;
@@ -298,83 +299,120 @@ private void updateSystemVmTemplates(DbUpgrade[] upgrades) {
     }
 
     protected void upgrade(CloudStackVersion dbVersion, CloudStackVersion currentVersion) {
+        executeProcedureScripts();
+        final DbUpgrade[] upgrades = executeUpgrades(dbVersion, currentVersion);
+
+        executeViewScripts();
+        updateSystemVmTemplates(upgrades);
+    }
+
+    protected void executeProcedureScripts() {
+        LOGGER.info(String.format("Executing Stored Procedure scripts that are under resource directory [%s].", PROCEDURES_DIRECTORY));
+        List<String> filesPathUnderViewsDirectory = FileUtil.getFilesPathsUnderResourceDirectory(PROCEDURES_DIRECTORY);
+
+        try (TransactionLegacy txn = TransactionLegacy.open("execute-procedure-scripts")) {
+            Connection conn = txn.getConnection();
+
+            for (String filePath : filesPathUnderViewsDirectory) {
+                LOGGER.debug(String.format("Executing PROCEDURE script [%s].", filePath));
+
+                InputStream viewScript = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);
+                runScript(conn, viewScript);
+            }
+
+            LOGGER.info(String.format("Finished execution of PROCEDURE scripts that are under resource directory [%s].", PROCEDURES_DIRECTORY));
+        } catch (SQLException e) {
+            String message = String.format("Unable to execute PROCEDURE scripts due to [%s].", e.getMessage());
+            LOGGER.error(message, e);
+            throw new CloudRuntimeException(message, e);
+        }
+    }
+
+    private DbUpgrade[] executeUpgrades(CloudStackVersion dbVersion, CloudStackVersion currentVersion) {
         LOGGER.info("Database upgrade must be performed from " + dbVersion + " to " + currentVersion);
 
         final DbUpgrade[] upgrades = calculateUpgradePath(dbVersion, currentVersion);
 
         for (DbUpgrade upgrade : upgrades) {
-            VersionVO version;
-            LOGGER.debug("Running upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
-                .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
-            TransactionLegacy txn = TransactionLegacy.open("Upgrade");
-            txn.start();
-            try {
-                Connection conn;
-                try {
-                    conn = txn.getConnection();
-                } catch (SQLException e) {
-                    String errorMessage = "Unable to upgrade the database";
-                    LOGGER.error(errorMessage, e);
-                    throw new CloudRuntimeException(errorMessage, e);
-                }
-                InputStream[] scripts = upgrade.getPrepareScripts();
-                if (scripts != null) {
-                    for (InputStream script : scripts) {
-                        runScript(conn, script);
-                    }
-                }
-
-                upgrade.performDataMigration(conn);
-
-                version = new VersionVO(upgrade.getUpgradedVersion());
-                version = _dao.persist(version);
+            VersionVO version = executeUpgrade(upgrade);
+            executeUpgradeCleanup(upgrade, version);
+        }
+        return upgrades;
+    }
 
-                txn.commit();
-            } catch (CloudRuntimeException e) {
+    private VersionVO executeUpgrade(DbUpgrade upgrade) {
+        VersionVO version;
+        LOGGER.debug("Running upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
+            .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+        TransactionLegacy txn = TransactionLegacy.open("Upgrade");
+        txn.start();
+        try {
+            Connection conn;
+            try {
+                conn = txn.getConnection();
+            } catch (SQLException e) {
                 String errorMessage = "Unable to upgrade the database";
                 LOGGER.error(errorMessage, e);
                 throw new CloudRuntimeException(errorMessage, e);
-            } finally {
-                txn.close();
+            }
+            InputStream[] scripts = upgrade.getPrepareScripts();
+            if (scripts != null) {
+                for (InputStream script : scripts) {
+                    runScript(conn, script);
+                }
             }
 
-            // Run the corresponding '-cleanup.sql' script
-            txn = TransactionLegacy.open("Cleanup");
-            try {
-                LOGGER.info("Cleanup upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
-                    .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+            upgrade.performDataMigration(conn);
 
-                txn.start();
-                Connection conn;
-                try {
-                    conn = txn.getConnection();
-                } catch (SQLException e) {
-                    LOGGER.error("Unable to cleanup the database", e);
-                    throw new CloudRuntimeException("Unable to cleanup the database", e);
-                }
+            version = new VersionVO(upgrade.getUpgradedVersion());
+            version = _dao.persist(version);
 
-                InputStream[] scripts = upgrade.getCleanupScripts();
-                if (scripts != null) {
-                    for (InputStream script : scripts) {
-                        runScript(conn, script);
-                        LOGGER.debug("Cleanup script " + upgrade.getClass().getSimpleName() + " is executed successfully");
-                    }
-                }
-                txn.commit();
+            txn.commit();
+        } catch (CloudRuntimeException e) {
+            String errorMessage = "Unable to upgrade the database";
+            LOGGER.error(errorMessage, e);
+            throw new CloudRuntimeException(errorMessage, e);
+        } finally {
+            txn.close();
+        }
+        return version;
+    }
 
-                txn.start();
-                version.setStep(Step.Complete);
-                version.setUpdated(new Date());
-                _dao.update(version.getId(), version);
-                txn.commit();
-                LOGGER.debug("Upgrade completed for version " + version.getVersion());
-            } finally {
-                txn.close();
+    private void executeUpgradeCleanup(DbUpgrade upgrade, VersionVO version) {
+        TransactionLegacy txn;
+        // Run the corresponding '-cleanup.sql' script
+        txn = TransactionLegacy.open("Cleanup");
+        try {
+            LOGGER.info("Cleanup upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
+                .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+
+            txn.start();
+            Connection conn;
+            try {
+                conn = txn.getConnection();
+            } catch (SQLException e) {
+                LOGGER.error("Unable to cleanup the database", e);
+                throw new CloudRuntimeException("Unable to cleanup the database", e);
             }
-        }
 
-        executeViewScripts();
-        updateSystemVmTemplates(upgrades);
+            InputStream[] scripts = upgrade.getCleanupScripts();
+            if (scripts != null) {
+                for (InputStream script : scripts) {
+                    runScript(conn, script);
+                    LOGGER.debug("Cleanup script " + upgrade.getClass().getSimpleName() + " is executed successfully");
+                }
+            }
+            txn.commit();
+
+            txn.start();
+            version.setStep(Step.Complete);
+            version.setUpdated(new Date());
+            _dao.update(version.getId(), version);
+            txn.commit();
+            LOGGER.debug("Upgrade completed for version " + version.getVersion());
+        } finally {
+            txn.close();
+        }
     }
 
     protected void executeViewScripts() {

engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java

@@ -38,4 +38,7 @@ public interface AffinityGroupDao extends GenericDao<AffinityGroupVO, Long> {
     AffinityGroupVO findByAccountAndType(Long accountId, String string);
 
     AffinityGroupVO findDomainLevelGroupByType(Long domainId, String string);
+
+    List<AffinityGroupVO> listByIds(List<Long> ids, boolean exclusive);
+
 }

engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java

@@ -31,6 +31,7 @@
 import com.cloud.utils.db.SearchCriteria;
 
 public class AffinityGroupDaoImpl extends GenericDaoBase<AffinityGroupVO, Long> implements AffinityGroupDao {
+    private SearchBuilder<AffinityGroupVO> IdsSearch;
     private SearchBuilder<AffinityGroupVO> AccountIdSearch;
     private SearchBuilder<AffinityGroupVO> AccountIdNameSearch;
     private SearchBuilder<AffinityGroupVO> AccountIdNamesSearch;
@@ -47,6 +48,10 @@ public AffinityGroupDaoImpl() {
 
     @PostConstruct
     protected void init() {
+        IdsSearch = createSearchBuilder();
+        IdsSearch.and("idIn", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
+        IdsSearch.done();
+
         AccountIdSearch = createSearchBuilder();
         AccountIdSearch.and("accountId", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
         AccountIdSearch.done();
@@ -158,4 +163,11 @@ public AffinityGroupVO findDomainLevelGroupByType(Long domainId, String type) {
         sc.setJoinParameters("domainTypeSearch", "domainId", domainId);
         return findOneBy(sc);
     }
+
+    @Override
+    public List<AffinityGroupVO> listByIds(List<Long> ids, boolean exclusive) {
+        SearchCriteria<AffinityGroupVO> sc = IdsSearch.create();
+        sc.setParameters("idIn", ids.toArray());
+        return lockRows(sc, null, exclusive);
+    }
 }

engine/schema/src/main/resources/META-INF/db/procedures/cloud.add_guest_os_and_hypervisor_mapping.sql

@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- PR#4699 Drop the procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` if it already exist.
+DROP PROCEDURE IF EXISTS `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING`;
+
+-- PR#4699 Create the procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` to add guest_os and guest_os_hypervisor mapping.
+CREATE PROCEDURE `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` (
+    IN guest_os_category_id bigint(20) unsigned,
+    IN guest_os_display_name VARCHAR(255),
+    IN guest_os_hypervisor_hypervisor_type VARCHAR(32),
+    IN guest_os_hypervisor_hypervisor_version VARCHAR(32),
+    IN guest_os_hypervisor_guest_os_name VARCHAR(255)
+        )
+BEGIN
+INSERT  INTO cloud.guest_os (uuid, category_id, display_name, created)
+SELECT 	UUID(), guest_os_category_id, guest_os_display_name, now()
+FROM    DUAL
+WHERE 	not exists( SELECT  1
+                     FROM    cloud.guest_os
+                     WHERE   cloud.guest_os.category_id = guest_os_category_id
+                       AND     cloud.guest_os.display_name = guest_os_display_name)
+
+;	INSERT  INTO cloud.guest_os_hypervisor (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created)
+     SELECT 	UUID(), guest_os_hypervisor_hypervisor_type, guest_os_hypervisor_hypervisor_version, guest_os_hypervisor_guest_os_name, guest_os.id, now()
+     FROM 	cloud.guest_os
+     WHERE 	guest_os.category_id = guest_os_category_id
+       AND 	guest_os.display_name = guest_os_display_name
+       AND	NOT EXISTS (SELECT  1
+                          FROM    cloud.guest_os_hypervisor as hypervisor
+                          WHERE   hypervisor_type = guest_os_hypervisor_hypervisor_type
+                            AND     hypervisor_version = guest_os_hypervisor_hypervisor_version
+                            AND     hypervisor.guest_os_id = guest_os.id
+                            AND     hypervisor.guest_os_name = guest_os_hypervisor_guest_os_name)
+;END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_column.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in cloud
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
+        IN in_table_name VARCHAR(200),
+		IN in_column_name VARCHAR(200),
+		IN in_column_definition VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_key.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_KEY`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_KEY` (
+		IN in_index_name VARCHAR(200)
+    , IN in_table_name VARCHAR(200)
+    , IN in_key_definition VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' ADD KEY ') ; SET @ddl = CONCAT(@ddl, ' ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', in_key_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_unique_index.sql

@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Idempotent ADD UNIQUE INDEX
+DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`;
+CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX` (
+    IN in_table_name VARCHAR(200)
+, IN in_index_name VARCHAR(200)
+, IN in_index_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD UNIQUE INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', in_index_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_unique_key.sql

@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Idempotent ADD UNIQUE KEY
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`;
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY` (
+    IN in_table_name VARCHAR(200)
+, IN in_key_name VARCHAR(200)
+, IN in_key_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD UNIQUE KEY ', in_key_name); SET @ddl = CONCAT(@ddl, ' ', in_key_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;