Skip to content

Commit 9830bbe

Browse files
rohityadavcloudrohityadavcloud
committed
framework/db: introduce a new MySQL table based distributed lock
This introduces a MySQL innodb table based distributed lock which can be used by one or more management server and its threads. This removes usage of MySQL server provided locking functions (GET_LOCK, RELEASE_LOCK) which are not replicated or supported currently by any MySQL clustering solutions. This would be the first main step in having CloudStack to work with a MySQL clustering solution such as InnoDB cluster, Percona Xtradb cluster, MariaDB galera cluster. There may be other changes required which can be found in due course if this feature works at scale. Signed-off-by: Rohit Yadav <[email protected]>
1 parent f965118 commit 9830bbe

File tree

4 files changed

+76
-42
lines changed

4 files changed

+76
-42
lines changed

‎engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
import com.cloud.utils.FileUtil;
3737
import org.apache.cloudstack.utils.CloudStackVersion;
38+
import org.apache.cloudstack.utils.identity.ManagementServerNode;
3839
import org.apache.commons.lang3.StringUtils;
3940
import org.apache.log4j.Logger;
4041

@@ -398,6 +399,7 @@ protected void executeViewScripts() {
398399

399400
@Override
400401
public void check() {
402+
initDistributedLock();
401403
GlobalLock lock = GlobalLock.getInternLock("DatabaseUpgrade");
402404
try {
403405
s_logger.info("Grabbing lock to check for database upgrade.");
@@ -441,6 +443,39 @@ public void check() {
441443
}
442444
}
443445

446+
private void initDistributedLock() {
447+
s_logger.info("Setting up distributed lock table if not created.");
448+
TransactionLegacy txn = TransactionLegacy.open("initDistributedLock");
449+
txn.start();
450+
String errorMessage = "Unable to get the database connections";
451+
try {
452+
Connection conn = txn.getConnection();
453+
errorMessage = "Unable to create distributed_lock table in the 'cloud' database ";
454+
String sql = "CREATE TABLE IF NOT EXISTS `cloud`.`distributed_lock` (" +
455+
" `name` varchar(1024) NOT NULL," +
456+
" `thread` varchar(1024) NOT NULL," +
457+
" `ms_id` bigint NOT NULL, `pid` int NOT NULL," +
458+
" `created` datetime DEFAULT NULL," +
459+
" PRIMARY KEY (`name`)," +
460+
" UNIQUE KEY `name` (`name`)" +
461+
") ENGINE=InnoDB DEFAULT CHARSET=utf8";
462+
try (PreparedStatement pstmt = conn.prepareStatement(sql)) {
463+
pstmt.execute();
464+
}
465+
try (PreparedStatement pstmt = conn.prepareStatement("DELETE FROM cloud.distributed_lock WHERE ms_id=?")) {
466+
pstmt.setLong(1, ManagementServerNode.getManagementServerId());
467+
pstmt.execute();
468+
}
469+
txn.commit();
470+
} catch (CloudRuntimeException | SQLException e) {
471+
s_logger.error(e.getMessage());
472+
errorMessage = String.format("%s due to %s.", errorMessage, e.getMessage());
473+
throw new CloudRuntimeException(errorMessage, e);
474+
} finally {
475+
txn.close();
476+
}
477+
}
478+
444479
private void initializeDatabaseEncryptors() {
445480
TransactionLegacy txn = TransactionLegacy.open("initializeDatabaseEncryptors");
446481
txn.start();

‎framework/db/src/main/java/com/cloud/utils/db/DbUtil.java‎

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import javax.persistence.Table;
4242
import javax.persistence.Transient;
4343

44+
import org.apache.cloudstack.utils.identity.ManagementServerNode;
4445
import org.apache.log4j.Logger;
4546

4647
import static com.cloud.utils.AutoCloseableUtil.closeAutoCloseable;
@@ -198,28 +199,36 @@ public static final String getTableName(Class<?> clazz) {
198199
public static boolean getGlobalLock(String name, int timeoutSeconds) {
199200
Connection conn = getConnectionForGlobalLocks(name, true);
200201
if (conn == null) {
201-
LOGGER.error("Unable to acquire DB connection for global lock system");
202+
LOGGER.error("Unable to acquire DB connection for distributed lock: " + name);
202203
return false;
203204
}
204205

205-
try (PreparedStatement pstmt = conn.prepareStatement("SELECT COALESCE(GET_LOCK(?, ?),0)");) {
206-
pstmt.setString(1, name);
207-
pstmt.setInt(2, timeoutSeconds);
208-
209-
try (ResultSet rs = pstmt.executeQuery();) {
210-
if (rs != null && rs.first()) {
211-
if (rs.getInt(1) > 0) {
212-
return true;
213-
} else {
214-
if (LOGGER.isDebugEnabled())
215-
LOGGER.debug("GET_LOCK() timed out on lock : " + name);
216-
}
206+
int remainingTime = timeoutSeconds;
207+
while (remainingTime > 0) {
208+
try (PreparedStatement pstmt = conn.prepareStatement(
209+
"INSERT INTO cloud.distributed_lock (name, thread, ms_id, pid, created) " +
210+
"VALUES (?, ?, ?, ?, now()) ON DUPLICATE KEY UPDATE name=name")) {
211+
pstmt.setString(1, name);
212+
pstmt.setString(2, Thread.currentThread().getName());
213+
pstmt.setLong(3, ManagementServerNode.getManagementServerId());
214+
pstmt.setLong(4, ProcessHandle.current().pid());
215+
if (pstmt.executeUpdate() > 0) {
216+
return true;
217217
}
218+
} catch (SQLException e) {
219+
LOGGER.error("Inserting to cloud.distributed_lock query threw exception ", e);
220+
} catch (Throwable e) {
221+
LOGGER.error("Inserting to cloud.distributed_lock query threw exception ", e);
222+
}
223+
224+
if (LOGGER.isDebugEnabled()) {
225+
LOGGER.debug("Waiting, cloud.distributed_lock already has the lock: " + name);
226+
}
227+
remainingTime = remainingTime - 1;
228+
try {
229+
Thread.sleep(1000L);
230+
} catch (InterruptedException ignore) {
218231
}
219-
} catch (SQLException e) {
220-
LOGGER.error("GET_LOCK() throws exception ", e);
221-
} catch (Throwable e) {
222-
LOGGER.error("GET_LOCK() throws exception ", e);
223232
}
224233

225234
removeConnectionForGlobalLocks(name);
@@ -234,24 +243,20 @@ public static Class<?> getEntityBeanType(GenericDao<?, Long> dao) {
234243
public static boolean releaseGlobalLock(String name) {
235244
try (Connection conn = getConnectionForGlobalLocks(name, false);) {
236245
if (conn == null) {
237-
LOGGER.error("Unable to acquire DB connection for global lock system");
246+
LOGGER.error("Unable to acquire DB connection for distributed lock: " + name);
238247
assert (false);
239248
return false;
240249
}
241250

242-
try (PreparedStatement pstmt = conn.prepareStatement("SELECT COALESCE(RELEASE_LOCK(?), 0)");) {
251+
try (PreparedStatement pstmt = conn.prepareStatement("DELETE FROM cloud.distributed_lock WHERE name=?")) {
243252
pstmt.setString(1, name);
244-
try (ResultSet rs = pstmt.executeQuery();) {
245-
if (rs != null && rs.first()) {
246-
return rs.getInt(1) > 0;
247-
}
248-
LOGGER.error("releaseGlobalLock:RELEASE_LOCK() returns unexpected result");
253+
if (pstmt.executeUpdate() > 0) {
254+
return true;
249255
}
256+
LOGGER.warn("releaseGlobalLock: failed to remove cloud.distributed_lock lock which does not exist: " + name);
250257
}
251-
} catch (SQLException e) {
252-
LOGGER.error("RELEASE_LOCK() throws exception ", e);
253258
} catch (Throwable e) {
254-
LOGGER.error("RELEASE_LOCK() throws exception ", e);
259+
LOGGER.error("Removing cloud.distributed_lock lock threw exception ", e);
255260
}
256261
return false;
257262
}

‎framework/db/src/test/java/com/cloud/utils/DbUtilTest.java‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -151,28 +151,24 @@ public void cleanup() throws SecurityException, NoSuchFieldException, IllegalArg
151151
public void getGlobalLock() throws SQLException {
152152
Mockito.when(dataSource.getConnection()).thenReturn(connection);
153153
Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement);
154-
Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet);
155-
Mockito.when(resultSet.first()).thenReturn(true);
156-
Mockito.when(resultSet.getInt(1)).thenReturn(1);
154+
Mockito.when(preparedStatement.executeUpdate()).thenReturn(1);
157155
Assert.assertTrue(DbUtil.getGlobalLock("TEST", 600));
158156

159157
Mockito.verify(connection).prepareStatement(Matchers.anyString());
160158
Mockito.verify(preparedStatement).close();
161-
Mockito.verify(resultSet).close();
162159
}
163160

164161
@Test
165162
public void getGlobalLockTimeout() throws SQLException {
166163
Mockito.when(dataSource.getConnection()).thenReturn(connection);
167164
Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement);
168-
Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet);
169-
Mockito.when(resultSet.first()).thenReturn(true);
170-
Mockito.when(resultSet.getInt(1)).thenReturn(0);
171-
Assert.assertFalse(DbUtil.getGlobalLock("TEST", 600));
165+
Mockito.when(preparedStatement.executeUpdate()).thenReturn(0);
172166

173-
Mockito.verify(connection).prepareStatement(Matchers.anyString());
174-
Mockito.verify(preparedStatement).close();
175-
Mockito.verify(resultSet).close();
167+
final int tries = 2;
168+
Assert.assertFalse(DbUtil.getGlobalLock("TEST", tries));
169+
170+
Mockito.verify(connection, Mockito.times(tries)).prepareStatement(Matchers.anyString());
171+
Mockito.verify(preparedStatement, Mockito.times(tries)).close();
176172
Mockito.verify(connection).close();
177173

178174
// if any error happens, the connection map must be cleared
@@ -238,13 +234,10 @@ void releaseGlobalLockNotexisting() throws SQLException {
238234
@Test
239235
public void releaseGlobalLock() throws SQLException {
240236
Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement);
241-
Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet);
242-
Mockito.when(resultSet.first()).thenReturn(true);
243-
Mockito.when(resultSet.getInt(1)).thenReturn(1);
237+
Mockito.when(preparedStatement.executeUpdate()).thenReturn(1);
244238
connectionMap.put("testLock", connection);
245239
Assert.assertTrue(DbUtil.releaseGlobalLock("testLock"));
246240

247-
Mockito.verify(resultSet).close();
248241
Mockito.verify(preparedStatement).close();
249242
Mockito.verify(connection).close();
250243
Assert.assertFalse(connectionMap.containsKey("testLock"));

‎setup/db/create-schema.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ DROP TABLE IF EXISTS `cloud`.`image_data_store`;
195195
DROP TABLE IF EXISTS `cloud`.`vm_compute_tags`;
196196
DROP TABLE IF EXISTS `cloud`.`vm_root_disk_tags`;
197197
DROP TABLE IF EXISTS `cloud`.`vm_network_map`;
198+
DROP TABLE IF EXISTS `cloud`.`distributed_lock`;
198199

199200
CREATE TABLE `cloud`.`version` (
200201
`id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT COMMENT 'id',

0 commit comments

Comments
 (0)