Skip to content

Commit 26e6a86

Browse files
HastyshellHastyshell
committed
[test](tde) Add rotate with restart docker cases
1 parent 1f2e186 commit 26e6a86

File tree

3 files changed

+230
-1
lines changed

3 files changed

+230
-1
lines changed

‎fe/fe-enterprise/src/main/java/org/apache/doris/enterprise/KeyManager.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.catalog.Env;
2121
import org.apache.doris.common.Config;
22+
import org.apache.doris.common.util.DebugPointUtil;
2223
import org.apache.doris.common.util.MasterDaemon;
2324
import org.apache.doris.encryption.DataKeyMaterial;
2425
import org.apache.doris.encryption.EncryptionKey;
@@ -38,6 +39,7 @@
3839
import org.apache.logging.log4j.LogManager;
3940
import org.apache.logging.log4j.Logger;
4041

42+
import java.util.ArrayList;
4143
import java.util.Arrays;
4244
import java.util.Base64;
4345
import java.util.HashMap;
@@ -320,15 +322,35 @@ public void rotateRootKey(Map<String, String> properties) {
320322

321323
KeyOperationInfo opInfo = new KeyOperationInfo();
322324
opInfo.setRootKeyInfo(newRootKeyInfo);
323-
List<EncryptionKey> masterKeys = store.getMasterKeys();
325+
List<EncryptionKey> masterKeys = new ArrayList<>(store.getMasterKeys());
324326
for (EncryptionKey masterKey : masterKeys) {
325327
byte[] newCiphertext = rootKeyProvider.encrypt(masterKey.plaintext);
326328
masterKey.ciphertext = Base64.getEncoder().encodeToString(newCiphertext);
327329
masterKey.mtime = System.currentTimeMillis();
328330
opInfo.addMasterKey(masterKey);
331+
332+
if (DebugPointUtil.isEnable("KeyManager.stopAfterOneMasterKeyChanged")) {
333+
// wait a long time here to trigger restart
334+
sleep(100000);
335+
}
336+
}
337+
338+
if (DebugPointUtil.isEnable("KeyManager.stopAfterAllMasterKeyChanged")) {
339+
sleep(100000);
329340
}
341+
330342
opInfo.setOpType(KeyOPType.ROTATE_ROOT_KEY);
343+
331344
Env.getCurrentEnv().getEditLog().logOperateKey(opInfo);
345+
346+
if (DebugPointUtil.isEnable("KeyManager.stopAfterRotateEditLogWritten")) {
347+
sleep(100000);
348+
}
349+
350+
store.clearMasterKeys();
351+
store.getMasterKeys().addAll(masterKeys);
352+
} catch (InterruptedException e) {
353+
// ignore, only for debug point
332354
} finally {
333355
store.writeUnlock();
334356
}

‎regression-test/suites/tde/test_restart_when_rotating.groovy‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
import org.apache.doris.regression.suite.ClusterOptions
2+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
3+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
4+
import software.amazon.awssdk.regions.Region
5+
import software.amazon.awssdk.services.kms.KmsClient
6+
import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionRequest
7+
8+
suite("test_restart_when_rotating", "docker") {
9+
def tdeAlgorithm = ["AES256", "SM4"]
10+
def cloudMode = [/*false,*/ true]
11+
cloudMode.each { mode ->
12+
tdeAlgorithm.each { algorithm ->
13+
def options = new ClusterOptions()
14+
options.cloudMode = mode
15+
options.enableDebugPoints()
16+
options.feConfigs += [
17+
'cloud_cluster_check_interval_second=1',
18+
'sys_log_verbose_modules=org',
19+
"doris_tde_key_endpoint=${context.config.tdeKeyEndpoint}",
20+
"doris_tde_key_region=${context.config.tdeKeyRegion}",
21+
"doris_tde_key_provider=${context.config.tdeKeyProvider}",
22+
"doris_tde_algorithm=${algorithm}",
23+
"doris_tde_key_id=${context.config.tdeKeyId}"
24+
]
25+
options.tdeAk = context.config.tdeAk
26+
options.tdeSk = context.config.tdeSk
27+
options.enableDebugPoints()
28+
29+
docker(options) {
30+
def tblName = "test_restart_when_rotating"
31+
sql """ DROP TABLE IF EXISTS ${tblName} """
32+
sql """
33+
CREATE TABLE IF NOT EXISTS ${tblName} (
34+
`k` int NOT NULL,
35+
`v` varchar(10) NOT NULL)
36+
DUPLICATE KEY(`k`)
37+
DISTRIBUTED BY HASH(`k`) BUCKETS 8
38+
PROPERTIES (
39+
"replication_allocation" = "tag.location.default: 1"
40+
)
41+
""";
42+
43+
(1..10).each { i ->
44+
sql """ INSERT INTO ${tblName} VALUES (${i}, "${i}") """
45+
}
46+
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
47+
48+
def keys = sql """ SELECT * FROM information_schema.encryption_keys """
49+
{
50+
def credProvider = StaticCredentialsProvider.create(
51+
AwsBasicCredentials.create(context.config.tdeAk, context.config.tdeSk)
52+
);
53+
def client = KmsClient.builder()
54+
.region(Region.of(context.config.tdeKeyRegion))
55+
.endpointOverride(URI.create(context.config.tdeKeyEndpoint))
56+
.credentialsProvider(credProvider)
57+
.build();
58+
59+
def resp = client.createKey()
60+
def keyId = resp.keyMetadata().keyId()
61+
try {
62+
GetDebugPoint().enableDebugPointForAllFEs("KeyManager.stopAfterOneMasterKeyChanged")
63+
def t = Thread.start {
64+
try {
65+
sql """ ADMIN ROTATE TDE ROOT KEY PROPERTIES(
66+
"doris_tde_key_provider" = "aws_kms",
67+
"doris_tde_key_id" = "${keyId}",
68+
"doris_tde_key_endpoint" = "${context.config.tdeKeyEndpoint}",
69+
"doris_tde_key_region" = "${context.config.tdeKeyRegion}"
70+
)
71+
"""
72+
} catch (Exception ignored) {
73+
// do nothing
74+
}
75+
}
76+
sleep(3000)
77+
78+
cluster.restartFrontends()
79+
sleep(30000)
80+
context.reconnectFe()
81+
82+
(1..10).each { i ->
83+
sql """ INSERT INTO ${tblName} VALUES (${i}, "${i}") """
84+
}
85+
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
86+
87+
def newKeys = sql """ SELECT * FROM information_schema.encryption_keys """
88+
assertEquals(keys[0][6], newKeys[0][6])
89+
keys = newKeys
90+
} finally {
91+
// delete cmk id
92+
def deleteReq = ScheduleKeyDeletionRequest.builder().keyId(keyId).build();
93+
client.scheduleKeyDeletion((ScheduleKeyDeletionRequest) deleteReq)
94+
GetDebugPoint().disableDebugPointForAllFEs("KeyManager.stopAfterOneMasterKeyChanged")
95+
}
96+
}
97+
98+
{
99+
def credProvider = StaticCredentialsProvider.create(
100+
AwsBasicCredentials.create(context.config.tdeAk, context.config.tdeSk)
101+
);
102+
def client = KmsClient.builder()
103+
.region(Region.of(context.config.tdeKeyRegion))
104+
.endpointOverride(URI.create(context.config.tdeKeyEndpoint))
105+
.credentialsProvider(credProvider)
106+
.build();
107+
108+
def resp = client.createKey()
109+
def keyId = resp.keyMetadata().keyId()
110+
try {
111+
GetDebugPoint().enableDebugPointForAllFEs("KeyManager.stopAfterAllMasterKeyChanged")
112+
def t = Thread.start {
113+
try {
114+
sql """ ADMIN ROTATE TDE ROOT KEY PROPERTIES(
115+
"doris_tde_key_provider" = "aws_kms",
116+
"doris_tde_key_id" = "${keyId}",
117+
"doris_tde_key_endpoint" = "${context.config.tdeKeyEndpoint}",
118+
"doris_tde_key_region" = "${context.config.tdeKeyRegion}"
119+
)
120+
"""
121+
} catch (Exception ignored) {
122+
// do nothing
123+
}
124+
}
125+
sleep(3000)
126+
127+
cluster.restartFrontends()
128+
sleep(30000)
129+
context.reconnectFe()
130+
131+
(1..10).each { i ->
132+
sql """ INSERT INTO ${tblName} VALUES (${i}, "${i}") """
133+
}
134+
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
135+
136+
def newKeys = sql """ SELECT * FROM information_schema.encryption_keys """
137+
assertEquals(keys[0][6], newKeys[0][6])
138+
keys = newKeys
139+
} finally {
140+
// delete cmk id
141+
def deleteReq = ScheduleKeyDeletionRequest.builder().keyId(keyId).build();
142+
client.scheduleKeyDeletion((ScheduleKeyDeletionRequest) deleteReq)
143+
GetDebugPoint().disableDebugPointForAllFEs("KeyManager.stopAfterAllMasterKeyChanged")
144+
}
145+
}
146+
147+
{
148+
def credProvider = StaticCredentialsProvider.create(
149+
AwsBasicCredentials.create(context.config.tdeAk, context.config.tdeSk)
150+
);
151+
def client = KmsClient.builder()
152+
.region(Region.of(context.config.tdeKeyRegion))
153+
.endpointOverride(URI.create(context.config.tdeKeyEndpoint))
154+
.credentialsProvider(credProvider)
155+
.build();
156+
157+
def resp = client.createKey()
158+
def keyId = resp.keyMetadata().keyId()
159+
try {
160+
GetDebugPoint().enableDebugPointForAllFEs("KeyManager.stopAfterRotateEditLogWritten")
161+
def t = Thread.start {
162+
try {
163+
sql """ ADMIN ROTATE TDE ROOT KEY PROPERTIES(
164+
"doris_tde_key_provider" = "aws_kms",
165+
"doris_tde_key_id" = "${keyId}",
166+
"doris_tde_key_endpoint" = "${context.config.tdeKeyEndpoint}",
167+
"doris_tde_key_region" = "${context.config.tdeKeyRegion}"
168+
)
169+
"""
170+
} catch (Exception ignored) {
171+
// do nothing
172+
}
173+
}
174+
sleep(3000)
175+
176+
cluster.restartFrontends()
177+
sleep(30000)
178+
context.reconnectFe()
179+
180+
(1..10).each { i ->
181+
sql """ INSERT INTO ${tblName} VALUES (${i}, "${i}") """
182+
}
183+
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
184+
185+
def newKeys = sql """ SELECT * FROM information_schema.encryption_keys """
186+
assertNotEquals(keys[0][6], newKeys[0][6])
187+
} finally {
188+
// delete cmk id
189+
def deleteReq = ScheduleKeyDeletionRequest.builder().keyId(keyId).build();
190+
client.scheduleKeyDeletion((ScheduleKeyDeletionRequest) deleteReq)
191+
GetDebugPoint().disableDebugPointForAllFEs("KeyManager.stopAfterRotateEditLogWritten")
192+
}
193+
}
194+
195+
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
196+
}
197+
}
198+
}
199+
}

‎regression-test/suites/tde/test_rotate_root_key.groovy‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ suite("test_rotate_root_key", "docker") {
8181
exception("unknown properties")
8282
}
8383

84+
def keys = sql """ SELECT * FROM information_schema.encryption_keys """
85+
8486
(1..10).each { i ->
8587
sql """ INSERT INTO ${tblName} VALUES (${i}, "${i}") """
8688
}
@@ -121,6 +123,9 @@ suite("test_rotate_root_key", "docker") {
121123
}
122124

123125
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
126+
def newKeys = sql """ SELECT * FROM information_schema.encryption_keys """
127+
assertNotEquals(keys[0][6], newKeys[0][6])
128+
keys = newKeys
124129
} finally {
125130
// delete cmk id
126131
def deleteReq = ScheduleKeyDeletionRequest.builder().keyId(keyId).build();
@@ -153,12 +158,15 @@ suite("test_rotate_root_key", "docker") {
153158
}
154159

155160
qt_sql """ SELECT * FROM ${tblName} ORDER BY `k` """
161+
def newKeys = sql """ SELECT * FROM information_schema.encryption_keys """
162+
assertNotEquals(keys[0][6], newKeys[0][6])
156163
} finally {
157164
// delete cmk id
158165
def deleteReq = ScheduleKeyDeletionRequest.builder().keyId(keyId).build();
159166
client.scheduleKeyDeletion((ScheduleKeyDeletionRequest) deleteReq)
160167
}
161168
}
169+
cluster.restartBackends()
162170
cluster.restartFrontends()
163171
sleep(30000)
164172
context.reconnectFe()

0 commit comments

Comments
 (0)