Skip to content

Commit 494c56a

Browse files
DaanHooglandDaanHoogland
authored
check for active MSses before starting DB upgrade (#12140)
1 parent e1c48c3 commit 494c56a

File tree

3 files changed

+342
-34
lines changed

3 files changed

+342
-34
lines changed

‎engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java‎

Lines changed: 86 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,12 @@
123123
import com.cloud.utils.crypt.DBEncryptionUtil;
124124
import com.cloud.utils.db.GlobalLock;
125125
import com.cloud.utils.db.ScriptRunner;
126+
import com.cloud.utils.db.Transaction;
127+
import com.cloud.utils.db.TransactionCallback;
126128
import com.cloud.utils.db.TransactionLegacy;
129+
import com.cloud.utils.db.TransactionStatus;
127130
import com.cloud.utils.exception.CloudRuntimeException;
131+
128132
import com.google.common.annotations.VisibleForTesting;
129133

130134
public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
@@ -247,7 +251,6 @@ protected void runScript(Connection conn, InputStream file) {
247251
LOGGER.error("Unable to execute upgrade script", e);
248252
throw new CloudRuntimeException("Unable to execute upgrade script", e);
249253
}
250-
251254
}
252255

253256
@VisibleForTesting
@@ -448,43 +451,101 @@ public void check() {
448451
throw new CloudRuntimeException("Unable to acquire lock to check for database integrity.");
449452
}
450453

451-
try {
452-
initializeDatabaseEncryptors();
453-
454-
final CloudStackVersion dbVersion = CloudStackVersion.parse(_dao.getCurrentVersion());
455-
final String currentVersionValue = this.getClass().getPackage().getImplementationVersion();
454+
doUpgrades(lock);
455+
} finally {
456+
lock.releaseRef();
457+
}
458+
}
456459

457-
if (StringUtils.isBlank(currentVersionValue)) {
458-
return;
460+
boolean isStandalone() throws CloudRuntimeException {
461+
return Transaction.execute(new TransactionCallback<>() {
462+
@Override
463+
public Boolean doInTransaction(TransactionStatus status) {
464+
String sql = "SELECT COUNT(*) FROM `cloud`.`mshost` WHERE `state` = 'UP'";
465+
try (Connection conn = TransactionLegacy.getStandaloneConnection();
466+
PreparedStatement pstmt = conn.prepareStatement(sql);
467+
ResultSet rs = pstmt.executeQuery()) {
468+
if (rs.next()) {
469+
int count = rs.getInt(1);
470+
return count == 0;
471+
}
472+
} catch (SQLException e) {
473+
String errorMessage = "Unable to check if the management server is running in standalone mode.";
474+
LOGGER.error(errorMessage, e);
475+
return false;
476+
} catch (NullPointerException npe) {
477+
String errorMessage = "Unable to check if the management server is running in standalone mode. Not able to get a Database connection.";
478+
LOGGER.error(errorMessage, npe);
479+
return false;
459480
}
481+
return true;
482+
}
483+
});
484+
}
460485

461-
String csVersion = SystemVmTemplateRegistration.parseMetadataFile();
462-
final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion);
463-
final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue);
464-
SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease());
465-
SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease());
486+
@VisibleForTesting
487+
protected void doUpgrades(GlobalLock lock) {
488+
try {
489+
initializeDatabaseEncryptors();
466490

467-
LOGGER.info("DB version = " + dbVersion + " Code Version = " + currentVersion);
491+
final CloudStackVersion dbVersion = CloudStackVersion.parse(_dao.getCurrentVersion());
492+
final String currentVersionValue = getImplementationVersion();
468493

469-
if (dbVersion.compareTo(currentVersion) > 0) {
470-
throw new CloudRuntimeException("Database version " + dbVersion + " is higher than management software version " + currentVersionValue);
471-
}
494+
if (StringUtils.isBlank(currentVersionValue)) {
495+
return;
496+
}
472497

473-
if (dbVersion.compareTo(currentVersion) == 0) {
474-
LOGGER.info("DB version and code version matches so no upgrade needed.");
475-
return;
476-
}
498+
String csVersion = parseSystemVmMetadata();
499+
final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion);
500+
final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue);
501+
SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease());
502+
SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease());
503+
504+
LOGGER.info("DB version = " + dbVersion + " Code Version = " + currentVersion);
505+
506+
if (dbVersion.compareTo(currentVersion) > 0) {
507+
throw new CloudRuntimeException("Database version " + dbVersion + " is higher than management software version " + currentVersionValue);
508+
}
477509

510+
if (dbVersion.compareTo(currentVersion) == 0) {
511+
LOGGER.info("DB version and code version matches so no upgrade needed.");
512+
return;
513+
}
514+
515+
if (isStandalone()) {
478516
upgrade(dbVersion, currentVersion);
479-
} finally {
480-
lock.unlock();
517+
} else {
518+
String errorMessage = "Database upgrade is required but the management server is running in a clustered environment. " +
519+
"Please perform the database upgrade when the management server is not running in a clustered environment.";
520+
LOGGER.error(errorMessage);
521+
handleClusteredUpgradeRequired(); // allow tests to override behavior
481522
}
482523
} finally {
483-
lock.releaseRef();
524+
lock.unlock();
484525
}
485526
}
486527

487-
private void initializeDatabaseEncryptors() {
528+
/**
529+
* Hook that is called when an upgrade is required but the management server is clustered.
530+
* Default behavior is to exit the JVM, tests can override to throw instead.
531+
*/
532+
@VisibleForTesting
533+
protected void handleClusteredUpgradeRequired() {
534+
System.exit(5); // I would prefer ServerDaemon.abort(errorMessage) but that would create a dependency hell
535+
}
536+
537+
@VisibleForTesting
538+
protected String getImplementationVersion() {
539+
return this.getClass().getPackage().getImplementationVersion();
540+
}
541+
542+
@VisibleForTesting
543+
protected String parseSystemVmMetadata() {
544+
return SystemVmTemplateRegistration.parseMetadataFile();
545+
}
546+
547+
// Make this protected so tests can noop it out
548+
protected void initializeDatabaseEncryptors() {
488549
TransactionLegacy txn = TransactionLegacy.open("initializeDatabaseEncryptors");
489550
txn.start();
490551
String errorMessage = "Unable to get the database connections";

‎engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerDoUpgradesTest.java‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.upgrade;
18+
19+
import static org.junit.Assert.assertFalse;
20+
import static org.junit.Assert.assertTrue;
21+
22+
import com.cloud.upgrade.dao.VersionDao;
23+
import com.cloud.upgrade.dao.VersionDaoImpl;
24+
import com.cloud.upgrade.dao.VersionVO;
25+
import com.cloud.utils.db.GlobalLock;
26+
import org.junit.Test;
27+
28+
public class DatabaseUpgradeCheckerDoUpgradesTest {
29+
30+
static class StubVersionDao extends VersionDaoImpl implements VersionDao {
31+
private final String currentVersion;
32+
33+
StubVersionDao(String currentVersion) {
34+
this.currentVersion = currentVersion;
35+
}
36+
37+
@Override
38+
public VersionVO findByVersion(String version, VersionVO.Step step) {
39+
return null;
40+
}
41+
42+
@Override
43+
public String getCurrentVersion() {
44+
return currentVersion;
45+
}
46+
47+
}
48+
49+
private static class TestableChecker extends DatabaseUpgradeChecker {
50+
boolean initializeCalled = false;
51+
boolean upgradeCalled = false;
52+
boolean clusterHandlerCalled = false;
53+
String implVersionOverride = null;
54+
String sysVmMetadataOverride = "4.8.0";
55+
boolean standaloneOverride = true;
56+
57+
TestableChecker(String daoVersion) {
58+
// set a stub DAO
59+
this._dao = new StubVersionDao(daoVersion);
60+
}
61+
62+
@Override
63+
protected void initializeDatabaseEncryptors() {
64+
initializeCalled = true;
65+
// noop instead of doing DB work
66+
}
67+
68+
@Override
69+
protected String getImplementationVersion() {
70+
return implVersionOverride;
71+
}
72+
73+
@Override
74+
protected String parseSystemVmMetadata() {
75+
return sysVmMetadataOverride;
76+
}
77+
78+
@Override
79+
boolean isStandalone() {
80+
return standaloneOverride;
81+
}
82+
83+
@Override
84+
protected void upgrade(org.apache.cloudstack.utils.CloudStackVersion dbVersion, org.apache.cloudstack.utils.CloudStackVersion currentVersion) {
85+
upgradeCalled = true;
86+
}
87+
88+
@Override
89+
protected void handleClusteredUpgradeRequired() {
90+
clusterHandlerCalled = true;
91+
}
92+
}
93+
94+
@Test
95+
public void testDoUpgrades_noImplementationVersion_returnsEarly() {
96+
TestableChecker checker = new TestableChecker("4.8.0");
97+
checker.implVersionOverride = ""; // blank -> should return early
98+
99+
GlobalLock lock = GlobalLock.getInternLock("test-noimpl");
100+
try {
101+
// acquire lock so doUpgrades can safely call unlock in finally
102+
lock.lock(1);
103+
checker.doUpgrades(lock);
104+
} finally {
105+
// ensure lock released if still held
106+
lock.releaseRef();
107+
}
108+
109+
assertTrue("initializeDatabaseEncryptors should be called before returning", checker.initializeCalled);
110+
assertFalse("upgrade should not be called when implementation version is blank", checker.upgradeCalled);
111+
assertFalse("cluster handler should not be called", checker.clusterHandlerCalled);
112+
}
113+
114+
@Test
115+
public void testDoUpgrades_dbUpToDate_noUpgrade() {
116+
// DB version = code version -> no upgrade
117+
TestableChecker checker = new TestableChecker("4.8.1");
118+
checker.implVersionOverride = "4.8.1";
119+
checker.sysVmMetadataOverride = "4.8.1";
120+
121+
GlobalLock lock = GlobalLock.getInternLock("test-uptodate");
122+
try {
123+
lock.lock(1);
124+
checker.doUpgrades(lock);
125+
} finally {
126+
lock.releaseRef();
127+
}
128+
129+
assertTrue(checker.initializeCalled);
130+
assertFalse(checker.upgradeCalled);
131+
assertFalse(checker.clusterHandlerCalled);
132+
}
133+
134+
@Test
135+
public void testDoUpgrades_requiresUpgrade_standalone_invokesUpgrade() {
136+
TestableChecker checker = new TestableChecker("4.8.0");
137+
checker.implVersionOverride = "4.8.2"; // code is newer than DB
138+
checker.sysVmMetadataOverride = "4.8.2";
139+
checker.standaloneOverride = true;
140+
141+
GlobalLock lock = GlobalLock.getInternLock("test-upgrade-standalone");
142+
try {
143+
lock.lock(1);
144+
checker.doUpgrades(lock);
145+
} finally {
146+
lock.releaseRef();
147+
}
148+
149+
assertTrue(checker.initializeCalled);
150+
assertTrue("upgrade should be invoked in standalone mode", checker.upgradeCalled);
151+
assertFalse(checker.clusterHandlerCalled);
152+
}
153+
154+
@Test
155+
public void testDoUpgrades_requiresUpgrade_clustered_invokesHandler() {
156+
TestableChecker checker = new TestableChecker("4.8.0");
157+
checker.implVersionOverride = "4.8.2"; // code is newer than DB
158+
checker.sysVmMetadataOverride = "4.8.2";
159+
checker.standaloneOverride = false;
160+
161+
GlobalLock lock = GlobalLock.getInternLock("test-upgrade-clustered");
162+
try {
163+
lock.lock(1);
164+
checker.doUpgrades(lock);
165+
} finally {
166+
lock.releaseRef();
167+
}
168+
169+
assertTrue(checker.initializeCalled);
170+
assertFalse("upgrade should not be invoked in clustered mode", checker.upgradeCalled);
171+
assertTrue("cluster handler should be invoked in clustered mode", checker.clusterHandlerCalled);
172+
}
173+
}

0 commit comments

Comments
 (0)