Skip to content

Commit f128cd5

Browse files
zerolbsonyzerolbsony
committed
Merge branch 'master' into alter_column_datatype_draft
# Conflicts: # iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java
2 parents 4486400 + 1c6a2b0 commit f128cd5

File tree

352 files changed

+10792
-5331
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

352 files changed

+10792
-5331
lines changed

‎.github/workflows/cluster-it-1c1d1a.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,5 @@ jobs:
5959
uses: actions/upload-artifact@v4
6060
with:
6161
name: cluster-log-ainode-${{ matrix.os }}
62-
path: integration-test/target/ainode-logs
62+
path: integration-test/target/*-logs
6363
retention-days: 30

‎dependencies.json‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@
3232
"com.nimbusds:nimbus-jose-jwt",
3333
"com.nimbusds:oauth2-oidc-sdk",
3434
"com.sun.istack:istack-commons-runtime",
35-
"com.timecho.ratis:ratis-client",
36-
"com.timecho.ratis:ratis-common",
37-
"com.timecho.ratis:ratis-grpc",
38-
"com.timecho.ratis:ratis-metrics-api",
39-
"com.timecho.ratis:ratis-proto",
40-
"com.timecho.ratis:ratis-server",
41-
"com.timecho.ratis:ratis-server-api",
35+
"org.apache.ratis:ratis-client",
36+
"org.apache.ratis:ratis-common",
37+
"org.apache.ratis:ratis-grpc",
38+
"org.apache.ratis:ratis-metrics-api",
39+
"org.apache.ratis:ratis-proto",
40+
"org.apache.ratis:ratis-server",
41+
"org.apache.ratis:ratis-server-api",
4242
"com.zaxxer:HikariCP",
4343
"commons-cli:commons-cli",
4444
"commons-codec:commons-codec",

‎integration-test/src/assembly/mpp-test.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
</fileSet>
6464
<fileSet>
6565
<outputDirectory>lib</outputDirectory>
66-
<directory>${project.basedir}/../iotdb-core/ainode/dist/</directory>
66+
<directory>${project.basedir}/../iotdb-core/ainode/dist/ainode/</directory>
6767
<fileMode>0755</fileMode>
6868
</fileSet>
6969
</fileSets>

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -580,8 +580,8 @@ public CommonConfig setSubscriptionEnabled(boolean subscriptionEnabled) {
580580
}
581581

582582
@Override
583-
public CommonConfig setDefaultStorageGroupLevel(int defaultStorageGroupLevel) {
584-
setProperty("default_database_level", String.valueOf(defaultStorageGroupLevel));
583+
public CommonConfig setDefaultDatabaseLevel(int defaultDatabaseLevel) {
584+
setProperty("default_database_level", String.valueOf(defaultDatabaseLevel));
585585
return this;
586586
}
587587

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -602,9 +602,9 @@ public CommonConfig setSubscriptionEnabled(boolean subscriptionEnabled) {
602602
}
603603

604604
@Override
605-
public CommonConfig setDefaultStorageGroupLevel(int defaultStorageGroupLevel) {
606-
dnConfig.setDefaultStorageGroupLevel(defaultStorageGroupLevel);
607-
cnConfig.setDefaultStorageGroupLevel(defaultStorageGroupLevel);
605+
public CommonConfig setDefaultDatabaseLevel(int defaultDatabaseLevel) {
606+
dnConfig.setDefaultDatabaseLevel(defaultDatabaseLevel);
607+
cnConfig.setDefaultDatabaseLevel(defaultDatabaseLevel);
608608
return this;
609609
}
610610

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@
2525
import org.apache.tsfile.external.commons.io.file.PathUtils;
2626
import org.slf4j.Logger;
2727

28-
import java.io.BufferedWriter;
2928
import java.io.File;
30-
import java.io.FileWriter;
29+
import java.io.FileInputStream;
30+
import java.io.FileOutputStream;
3131
import java.io.IOException;
3232
import java.nio.file.Files;
3333
import java.nio.file.LinkOption;
@@ -37,6 +37,7 @@
3737
import java.nio.file.StandardCopyOption;
3838
import java.util.ArrayList;
3939
import java.util.List;
40+
import java.util.Properties;
4041
import java.util.stream.Stream;
4142

4243
import static org.apache.iotdb.it.env.cluster.ClusterConstant.AI_NODE_NAME;
@@ -62,15 +63,19 @@ public class AINodeWrapper extends AbstractNodeWrapper {
6263
public static final String CACHE_BUILT_IN_MODEL_PATH = "/data/ainode/models/weights";
6364

6465
private void replaceAttribute(String[] keys, String[] values, String filePath) {
65-
try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath, true))) {
66-
for (int i = 0; i < keys.length; i++) {
67-
String line = keys[i] + "=" + values[i];
68-
writer.newLine();
69-
writer.write(line);
70-
}
66+
Properties props = new Properties();
67+
try (FileInputStream in = new FileInputStream(filePath)) {
68+
props.load(in);
69+
} catch (IOException e) {
70+
logger.warn("Failed to load existing AINode properties from {}, because: ", filePath, e);
71+
}
72+
for (int i = 0; i < keys.length; i++) {
73+
props.setProperty(keys[i], values[i]);
74+
}
75+
try (FileOutputStream out = new FileOutputStream(filePath)) {
76+
props.store(out, "Updated by AINode integration-test env");
7177
} catch (IOException e) {
72-
logger.error(
73-
"Failed to set attribute for AINode in file: {} because {}", filePath, e.getMessage());
78+
logger.error("Failed to save properties to {}, because:", filePath, e);
7479
}
7580
}
7681

‎integration-test/src/main/java/org/apache/iotdb/itbase/constant/TestConstant.java‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,12 @@ public static String recordToInsert(TSRecord record) {
168168
}
169169

170170
public static String getTestTsFilePath(
171-
String logicalStorageGroupName,
172-
long VirtualStorageGroupId,
171+
String logicalDatabaseName,
172+
long VirtualDatabaseId,
173173
long TimePartitionId,
174174
long tsFileVersion) {
175175
String filePath =
176-
String.format(
177-
TEST_TSFILE_PATH, logicalStorageGroupName, VirtualStorageGroupId, TimePartitionId);
176+
String.format(TEST_TSFILE_PATH, logicalDatabaseName, VirtualDatabaseId, TimePartitionId);
178177
String fileName =
179178
System.currentTimeMillis()
180179
+ FilePathUtils.FILE_NAME_SEPARATOR
@@ -184,11 +183,8 @@ public static String getTestTsFilePath(
184183
}
185184

186185
public static String getTestTsFileDir(
187-
String logicalStorageGroupName, long VirtualStorageGroupId, long TimePartitionId) {
186+
String logicalDatabaseName, long VirtualDatabaseId, long TimePartitionId) {
188187
return String.format(
189-
TestConstant.TEST_TSFILE_PATH,
190-
logicalStorageGroupName,
191-
VirtualStorageGroupId,
192-
TimePartitionId);
188+
TestConstant.TEST_TSFILE_PATH, logicalDatabaseName, VirtualDatabaseId, TimePartitionId);
193189
}
194190
}

‎integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ CommonConfig setSubscriptionPrefetchTsFileBatchMaxSizeInBytes(
186186

187187
CommonConfig setSubscriptionEnabled(boolean subscriptionEnabled);
188188

189-
default CommonConfig setDefaultStorageGroupLevel(int defaultStorageGroupLevel) {
189+
default CommonConfig setDefaultDatabaseLevel(int defaultDatabaseLevel) {
190190
return this;
191191
}
192192

‎integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeConcurrentInferenceIT.java‎

Lines changed: 1 addition & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -90,33 +90,6 @@ private static void prepareDataForTableModel() throws SQLException {
9090
}
9191
}
9292

93-
// @Test
94-
public void concurrentCPUCallInferenceTest() throws SQLException, InterruptedException {
95-
concurrentCPUCallInferenceTest("timer_xl");
96-
concurrentCPUCallInferenceTest("sundial");
97-
}
98-
99-
private void concurrentCPUCallInferenceTest(String modelId)
100-
throws SQLException, InterruptedException {
101-
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
102-
Statement statement = connection.createStatement()) {
103-
final int threadCnt = 4;
104-
final int loop = 10;
105-
final int predictLength = 96;
106-
statement.execute(String.format("LOAD MODEL %s TO DEVICES 'cpu'", modelId));
107-
checkModelOnSpecifiedDevice(statement, modelId, "cpu");
108-
concurrentInference(
109-
statement,
110-
String.format(
111-
"CALL INFERENCE(%s, 'SELECT s FROM root.AI', predict_length=%d)",
112-
modelId, predictLength),
113-
threadCnt,
114-
loop,
115-
predictLength);
116-
statement.execute(String.format("UNLOAD MODEL %s FROM DEVICES 'cpu'", modelId));
117-
}
118-
}
119-
12093
// @Test
12194
public void concurrentGPUCallInferenceTest() throws SQLException, InterruptedException {
12295
concurrentGPUCallInferenceTest("timer_xl");
@@ -150,39 +123,6 @@ private void concurrentGPUCallInferenceTest(String modelId)
150123
String forecastUDTFSql =
151124
"SELECT forecast(s, 'MODEL_ID'='%s', 'PREDICT_LENGTH'='%d') FROM root.AI";
152125

153-
@Test
154-
public void concurrentCPUForecastTest() throws SQLException, InterruptedException {
155-
concurrentCPUForecastTest("timer_xl", forecastUDTFSql);
156-
concurrentCPUForecastTest("sundial", forecastUDTFSql);
157-
concurrentCPUForecastTest("timer_xl", forecastTableFunctionSql);
158-
concurrentCPUForecastTest("sundial", forecastTableFunctionSql);
159-
}
160-
161-
private void concurrentCPUForecastTest(String modelId, String selectSQL)
162-
throws SQLException, InterruptedException {
163-
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
164-
Statement statement = connection.createStatement()) {
165-
final int threadCnt = 4;
166-
final int loop = 10;
167-
final int predictLength = 96;
168-
statement.execute(String.format("LOAD MODEL %s TO DEVICES 'cpu'", modelId));
169-
checkModelOnSpecifiedDevice(statement, modelId, "cpu");
170-
long startTime = System.currentTimeMillis();
171-
concurrentInference(
172-
statement,
173-
String.format(selectSQL, modelId, predictLength),
174-
threadCnt,
175-
loop,
176-
predictLength);
177-
long endTime = System.currentTimeMillis();
178-
LOGGER.info(
179-
String.format(
180-
"Model %s concurrent inference %d reqs (%d threads, %d loops) in CPU takes time: %dms",
181-
modelId, threadCnt * loop, threadCnt, loop, endTime - startTime));
182-
statement.execute(String.format("UNLOAD MODEL %s FROM DEVICES 'cpu'", modelId));
183-
}
184-
}
185-
186126
@Test
187127
public void concurrentGPUForecastTest() throws SQLException, InterruptedException {
188128
concurrentGPUForecastTest("timer_xl", forecastUDTFSql);
@@ -221,7 +161,7 @@ private void checkModelOnSpecifiedDevice(Statement statement, String modelId, St
221161
throws SQLException, InterruptedException {
222162
Set<String> targetDevices = ImmutableSet.copyOf(device.split(","));
223163
LOGGER.info("Checking model: {} on target devices: {}", modelId, targetDevices);
224-
for (int retry = 0; retry < 20; retry++) {
164+
for (int retry = 0; retry < 200; retry++) {
225165
Set<String> foundDevices = new HashSet<>();
226166
try (final ResultSet resultSet =
227167
statement.executeQuery(String.format("SHOW LOADED MODELS '%s'", device))) {

‎integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public void tearDown() {
106106
@Test
107107
public void testPartitionInfoSnapshot() throws Exception {
108108
final String sg = "root.sg";
109-
final int storageGroupNum = 10;
109+
final int databaseNum = 10;
110110
final int seriesPartitionSlotsNum = 10;
111111
final int timePartitionSlotsNum = 10;
112112

@@ -118,18 +118,18 @@ public void testPartitionInfoSnapshot() throws Exception {
118118

119119
Set<TCQEntry> expectedCQEntries = createCQs(client);
120120

121-
for (int i = 0; i < storageGroupNum; i++) {
122-
String storageGroup = sg + i;
123-
TDatabaseSchema storageGroupSchema = new TDatabaseSchema(storageGroup);
124-
TSStatus status = client.setDatabase(storageGroupSchema);
121+
for (int i = 0; i < databaseNum; i++) {
122+
String database = sg + i;
123+
TDatabaseSchema databaseSchema = new TDatabaseSchema(database);
124+
TSStatus status = client.setDatabase(databaseSchema);
125125
assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
126126

127127
for (int j = 0; j < seriesPartitionSlotsNum; j++) {
128128
TSeriesPartitionSlot seriesPartitionSlot = new TSeriesPartitionSlot(j);
129129

130130
// Create SchemaPartition
131131
ByteBuffer patternTree =
132-
generatePatternTreeBuffer(new String[] {storageGroup + ".d" + j + ".s"});
132+
generatePatternTreeBuffer(new String[] {database + ".d" + j + ".s"});
133133
TSchemaPartitionReq schemaPartitionReq = new TSchemaPartitionReq(patternTree);
134134
TSchemaPartitionTableResp schemaPartitionTableResp =
135135
client.getOrCreateSchemaPartitionTable(schemaPartitionReq);
@@ -139,10 +139,8 @@ public void testPartitionInfoSnapshot() throws Exception {
139139
schemaPartitionTableResp.getStatus().getCode());
140140
Assert.assertNotNull(schemaPartitionTableResp.getSchemaPartitionTable());
141141
assertEquals(1, schemaPartitionTableResp.getSchemaPartitionTableSize());
142-
Assert.assertNotNull(
143-
schemaPartitionTableResp.getSchemaPartitionTable().get(storageGroup));
144-
assertEquals(
145-
1, schemaPartitionTableResp.getSchemaPartitionTable().get(storageGroup).size());
142+
Assert.assertNotNull(schemaPartitionTableResp.getSchemaPartitionTable().get(database));
143+
assertEquals(1, schemaPartitionTableResp.getSchemaPartitionTable().get(database).size());
146144

147145
for (int k = 0; k < timePartitionSlotsNum; k++) {
148146
TTimePartitionSlot timePartitionSlot =
@@ -151,9 +149,9 @@ public void testPartitionInfoSnapshot() throws Exception {
151149
// Create DataPartition
152150
Map<String, Map<TSeriesPartitionSlot, TTimeSlotList>> partitionSlotsMap =
153151
new HashMap<>();
154-
partitionSlotsMap.put(storageGroup, new HashMap<>());
152+
partitionSlotsMap.put(database, new HashMap<>());
155153
partitionSlotsMap
156-
.get(storageGroup)
154+
.get(database)
157155
.put(
158156
seriesPartitionSlot,
159157
new TTimeSlotList()
@@ -167,19 +165,18 @@ public void testPartitionInfoSnapshot() throws Exception {
167165
dataPartitionTableResp.getStatus().getCode());
168166
Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable());
169167
assertEquals(1, dataPartitionTableResp.getDataPartitionTableSize());
170-
Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable().get(storageGroup));
171-
assertEquals(
172-
1, dataPartitionTableResp.getDataPartitionTable().get(storageGroup).size());
168+
Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable().get(database));
169+
assertEquals(1, dataPartitionTableResp.getDataPartitionTable().get(database).size());
173170
Assert.assertNotNull(
174171
dataPartitionTableResp
175172
.getDataPartitionTable()
176-
.get(storageGroup)
173+
.get(database)
177174
.get(seriesPartitionSlot));
178175
assertEquals(
179176
1,
180177
dataPartitionTableResp
181178
.getDataPartitionTable()
182-
.get(storageGroup)
179+
.get(database)
183180
.get(seriesPartitionSlot)
184181
.size());
185182
}

0 commit comments

Comments
 (0)