Skip to content

Commit 23be220

Browse files
luoluoyuyuluoluoyuyu
authored
perf: Optimize wide table write performance (apache#16699)
* Refactor TableSchema handling to reduce conversion overhead * update * update * update * update * update * update * update * update * fix * update semanticCheck * fix: cache table schema by database and table * spotless * update * update * update * update * update * fix InsertStatementTest * update * update * update * update * Revert "update" This reverts commit 7eb3b3a. * ignore object type * update * update * update * update * update * update * update * update * update * update * update * update * update * fix it * fix it * fix * fix * fix * fix * fix * fix * update * fix * fix * fix * fix * fix * fix * fix * fix
1 parent 8cb4b9b commit 23be220

File tree

36 files changed

+2832
-322
lines changed

36 files changed

+2832
-322
lines changed

‎integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.apache.iotdb.db.it.selectinto;
2121

22+
import org.apache.iotdb.db.it.utils.TSDataTypeTestUtils;
2223
import org.apache.iotdb.it.env.EnvFactory;
2324
import org.apache.iotdb.it.framework.IoTDBTestRunner;
2425
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -104,13 +105,11 @@ public class IoTDBSelectIntoIT {
104105
static {
105106
SELECT_INTO_SQL_LIST.add("CREATE DATABASE root.sg_type");
106107
for (int deviceId = 0; deviceId < 6; deviceId++) {
107-
for (TSDataType dataType : TSDataType.values()) {
108-
if (!dataType.equals(TSDataType.VECTOR) && !dataType.equals(TSDataType.UNKNOWN)) {
109-
SELECT_INTO_SQL_LIST.add(
110-
String.format(
111-
"CREATE TIMESERIES root.sg_type.d_%d.s_%s %s",
112-
deviceId, dataType.name().toLowerCase(), dataType));
113-
}
108+
for (TSDataType dataType : TSDataTypeTestUtils.getSupportedTypes()) {
109+
SELECT_INTO_SQL_LIST.add(
110+
String.format(
111+
"CREATE TIMESERIES root.sg_type.d_%d.s_%s %s",
112+
deviceId, dataType.name().toLowerCase(), dataType));
114113
}
115114
}
116115
for (int time = 0; time < 12; time++) {

‎integration-test/src/test/java/org/apache/iotdb/db/it/utils/TSDataTypeTestUtils.java‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iotdb.db.it.utils;
20+
21+
import org.apache.tsfile.enums.TSDataType;
22+
23+
import java.util.Arrays;
24+
import java.util.HashSet;
25+
import java.util.List;
26+
import java.util.Set;
27+
import java.util.stream.Collectors;
28+
29+
/**
30+
* Utility class for TSDataType operations in integration tests. This class provides helper methods
31+
* to filter out unsupported data types that should not be used in tests.
32+
*
33+
* <p>Usage in IT tests:
34+
*
35+
* <pre>{@code
36+
* Set<TSDataType> dataTypes = TSDataTypeTestUtils.getSupportedTypes();
37+
* for (TSDataType from : dataTypes) {
38+
* for (TSDataType to : dataTypes) {
39+
* // test logic
40+
* }
41+
* }
42+
* }</pre>
43+
*
44+
* <p>To find this utility class quickly, search for: "TSDataTypeTestUtils" or "getSupportedTypes"
45+
*/
46+
public class TSDataTypeTestUtils {
47+
48+
private TSDataTypeTestUtils() {
49+
// utility class
50+
}
51+
52+
/**
53+
* Get the set of unsupported TSDataType values that should be filtered out in tests.
54+
*
55+
* <p>Currently includes: VECTOR, UNKNOWN, OBJECT
56+
*
57+
* @return Set of unsupported TSDataType values
58+
*/
59+
public static Set<TSDataType> getUnsupportedTypes() {
60+
Set<TSDataType> unsupported = new HashSet<>();
61+
unsupported.add(TSDataType.VECTOR);
62+
unsupported.add(TSDataType.UNKNOWN);
63+
unsupported.add(TSDataType.OBJECT);
64+
return unsupported;
65+
}
66+
67+
/**
68+
* Check if a TSDataType is supported for general use (not an internal type).
69+
*
70+
* @param dataType the TSDataType to check
71+
* @return true if the type is supported, false otherwise
72+
*/
73+
public static boolean isSupportedType(TSDataType dataType) {
74+
return !getUnsupportedTypes().contains(dataType);
75+
}
76+
77+
/**
78+
* Get all supported TSDataType values (filters out unsupported types).
79+
*
80+
* <p>This method filters out VECTOR, UNKNOWN, and any other types returned by {@link
81+
* #getUnsupportedTypes()}.
82+
*
83+
* @return Set of supported TSDataType values
84+
*/
85+
public static Set<TSDataType> getSupportedTypes() {
86+
Set<TSDataType> allTypes = new HashSet<>(Arrays.asList(TSDataType.values()));
87+
allTypes.removeAll(getUnsupportedTypes());
88+
return allTypes;
89+
}
90+
91+
/**
92+
* Get all supported TSDataType values as a List (filters out unsupported types).
93+
*
94+
* @return List of supported TSDataType values
95+
*/
96+
public static List<TSDataType> getSupportedTypesList() {
97+
return Arrays.stream(TSDataType.values())
98+
.filter(TSDataTypeTestUtils::isSupportedType)
99+
.collect(Collectors.toList());
100+
}
101+
102+
/**
103+
* Filter a collection of TSDataType values to only include supported types.
104+
*
105+
* @param dataTypes collection of TSDataType values to filter
106+
* @return Set containing only supported types
107+
*/
108+
public static Set<TSDataType> filterSupportedTypes(Set<TSDataType> dataTypes) {
109+
Set<TSDataType> result = new HashSet<>(dataTypes);
110+
result.removeAll(getUnsupportedTypes());
111+
return result;
112+
}
113+
}

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public static void insertData(
108108
for (int i = startInclusive; i < endExclusive; ++i) {
109109
list.add(
110110
String.format(
111-
"insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
111+
"insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
112112
tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i));
113113
}
114114
list.add("flush");
@@ -128,7 +128,7 @@ public static void insertData(
128128
for (int i = startInclusive; i < endExclusive; ++i) {
129129
list.add(
130130
String.format(
131-
"insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
131+
"insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
132132
tableName,
133133
deviceIndex,
134134
deviceIndex,
@@ -208,7 +208,7 @@ public static boolean insertDataNotThrowError(
208208
for (int i = start; i < end; ++i) {
209209
list.add(
210210
String.format(
211-
"insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
211+
"insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
212212
tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i));
213213
}
214214
try {
@@ -230,7 +230,7 @@ public static boolean insertData(
230230
for (int i = start; i < end; ++i) {
231231
list.add(
232232
String.format(
233-
"insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
233+
"insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
234234
tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i));
235235
}
236236
list.add("flush");

‎integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ public void testPartialInsertTablet() {
294294
}
295295
Assert.assertEquals(200, cnt);
296296
}
297+
session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='true'");
297298
} catch (Exception e) {
298299
e.printStackTrace();
299300
fail(e.getMessage());
@@ -505,6 +506,7 @@ public void testInsertMultiRowWithMisMatchDataType() throws SQLException {
505506
public void testInsertMultiRowWithNull() throws SQLException {
506507
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
507508
Statement st1 = connection.createStatement()) {
509+
st1.execute("SET CONFIGURATION enable_auto_create_schema='false'");
508510
st1.execute("use \"test\"");
509511
st1.execute(
510512
"create table wf14 (tag1 string tag, status boolean field, temperature float field)");
@@ -519,6 +521,8 @@ public void testInsertMultiRowWithNull() throws SQLException {
519521
st2.execute("CREATE TABLE wf15 (wt string tag, s1 double field, s2 double field)");
520522
st2.execute(
521523
"INSERT INTO wf15(wt, time, s1) VALUES ('1', 6, 10),('1', 7,12),('1', 8,14),('1', 9,160),('1', 10,null),('1', 11,58)");
524+
525+
st2.execute("SET CONFIGURATION enable_auto_create_schema='true'");
522526
} catch (SQLException e) {
523527
fail(e.getMessage());
524528
}
@@ -829,6 +833,7 @@ public void testInsertCaseSensitivity()
829833
@Test
830834
public void testInsertKeyword() throws IoTDBConnectionException, StatementExecutionException {
831835
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
836+
session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='true'");
832837
session.executeNonQueryStatement("USE \"test\"");
833838
session.executeNonQueryStatement(
834839
"create table table20 ("
@@ -1066,6 +1071,7 @@ public void testInsertTabletWithTTL()
10661071
public void testInsertUnsequenceData()
10671072
throws IoTDBConnectionException, StatementExecutionException {
10681073
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
1074+
session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='true'");
10691075
session.executeNonQueryStatement("USE \"test\"");
10701076
// the table is missing column "m2"
10711077
session.executeNonQueryStatement(

‎integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.iotdb.relational.it.session;
2020

2121
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
22+
import org.apache.iotdb.db.it.utils.TSDataTypeTestUtils;
2223
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
2324
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
2425
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
@@ -64,7 +65,6 @@
6465
import java.util.ArrayList;
6566
import java.util.Arrays;
6667
import java.util.Collections;
67-
import java.util.HashSet;
6868
import java.util.List;
6969
import java.util.Set;
7070

@@ -1626,11 +1626,7 @@ private Object genValue(TSDataType dataType, int i) {
16261626
public void insertRelationalTabletWithAutoCastTest()
16271627
throws IoTDBConnectionException, StatementExecutionException {
16281628
int testNum = 14;
1629-
Set<TSDataType> dataTypes = new HashSet<>();
1630-
Collections.addAll(dataTypes, TSDataType.values());
1631-
dataTypes.remove(TSDataType.VECTOR);
1632-
dataTypes.remove(TSDataType.UNKNOWN);
1633-
dataTypes.remove(TSDataType.OBJECT);
1629+
Set<TSDataType> dataTypes = TSDataTypeTestUtils.getSupportedTypes();
16341630

16351631
try {
16361632
for (TSDataType from : dataTypes) {
@@ -1717,11 +1713,7 @@ public void dropTableOfTheSameNameTest()
17171713
public void insertRelationalRowWithAutoCastTest()
17181714
throws IoTDBConnectionException, StatementExecutionException {
17191715
int testNum = 17;
1720-
Set<TSDataType> dataTypes = new HashSet<>();
1721-
Collections.addAll(dataTypes, TSDataType.values());
1722-
dataTypes.remove(TSDataType.VECTOR);
1723-
dataTypes.remove(TSDataType.UNKNOWN);
1724-
dataTypes.remove(TSDataType.OBJECT);
1716+
Set<TSDataType> dataTypes = TSDataTypeTestUtils.getSupportedTypes();
17251717

17261718
for (TSDataType from : dataTypes) {
17271719
for (TSDataType to : dataTypes) {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,8 +1117,6 @@ public class IoTDBConfig {
11171117

11181118
private boolean loadActiveListeningEnable = true;
11191119

1120-
private long loadTableSchemaCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
1121-
11221120
private long loadMeasurementIdCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
11231121

11241122
private int loadTsFileSpiltPartitionMaxSize = 10;
@@ -4025,14 +4023,6 @@ public void setLoadActiveListeningEnable(boolean loadActiveListeningEnable) {
40254023
this.loadActiveListeningEnable = loadActiveListeningEnable;
40264024
}
40274025

4028-
public long getLoadTableSchemaCacheSizeInBytes() {
4029-
return loadTableSchemaCacheSizeInBytes;
4030-
}
4031-
4032-
public void setLoadTableSchemaCacheSizeInBytes(long loadTableSchemaCacheSizeInBytes) {
4033-
this.loadTableSchemaCacheSizeInBytes = loadTableSchemaCacheSizeInBytes;
4034-
}
4035-
40364026
public long getLoadMeasurementIdCacheSizeInBytes() {
40374027
return loadMeasurementIdCacheSizeInBytes;
40384028
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2342,12 +2342,6 @@ private void loadLoadTsFileProps(TrimProperties properties) {
23422342
? conf.getLoadActiveListeningCheckIntervalSeconds()
23432343
: loadActiveListeningCheckIntervalSeconds);
23442344

2345-
conf.setLoadTableSchemaCacheSizeInBytes(
2346-
Long.parseLong(
2347-
properties.getProperty(
2348-
"load_table_schema_cache_size_in_bytes",
2349-
Long.toString(conf.getLoadTableSchemaCacheSizeInBytes()))));
2350-
23512345
conf.setLoadMeasurementIdCacheSizeInBytes(
23522346
Long.parseLong(
23532347
properties.getProperty(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -760,9 +760,7 @@ private Scope visitInsert(WrappedInsertStatement insert, Optional<Scope> scope)
760760

761761
final MPPQueryContext context = insert.getContext();
762762
InsertBaseStatement innerInsert = insert.getInnerTreeStatement();
763-
764-
innerInsert.semanticCheck();
765-
innerInsert.toLowerCase();
763+
innerInsert.toLowerCaseForDevicePath();
766764

767765
innerInsert =
768766
AnalyzeUtils.analyzeInsert(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.iotdb.commons.partition.DataPartition;
2323
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
2424
import org.apache.iotdb.commons.partition.SchemaPartition;
25+
import org.apache.iotdb.commons.schema.table.InsertNodeMeasurementInfo;
2526
import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinWindowFunction;
2627
import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException;
2728
import org.apache.iotdb.db.exception.sql.SemanticException;
@@ -30,6 +31,7 @@
3031
import org.apache.iotdb.db.queryengine.plan.analyze.IModelFetcher;
3132
import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
3233
import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType;
34+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator;
3335
import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl;
3436
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
3537
import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException;
@@ -125,6 +127,14 @@ Optional<TableSchema> validateTableHeaderSchema(
125127
final boolean isStrictIdColumn)
126128
throws LoadAnalyzeTableColumnDisorderException;
127129

130+
void validateInsertNodeMeasurements(
131+
final String database,
132+
final InsertNodeMeasurementInfo measurementInfo,
133+
final MPPQueryContext context,
134+
final boolean allowCreateTable,
135+
final TableHeaderSchemaValidator.MeasurementValidator measurementValidator,
136+
final TableHeaderSchemaValidator.TagColumnHandler tagColumnHandler);
137+
128138
/**
129139
* This method is used for table device validation and should be invoked after column validation.
130140
*

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.iotdb.commons.partition.DataPartition;
2525
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
2626
import org.apache.iotdb.commons.partition.SchemaPartition;
27+
import org.apache.iotdb.commons.schema.table.InsertNodeMeasurementInfo;
2728
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
2829
import org.apache.iotdb.commons.schema.table.TsTable;
2930
import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction;
@@ -1397,6 +1398,24 @@ public Optional<TableSchema> validateTableHeaderSchema(
13971398
database, tableSchema, context, allowCreateTable, isStrictTagColumn);
13981399
}
13991400

1401+
@Override
1402+
public void validateInsertNodeMeasurements(
1403+
final String database,
1404+
final InsertNodeMeasurementInfo measurementInfo,
1405+
final MPPQueryContext context,
1406+
final boolean allowCreateTable,
1407+
final TableHeaderSchemaValidator.MeasurementValidator measurementValidator,
1408+
final TableHeaderSchemaValidator.TagColumnHandler tagColumnHandler) {
1409+
TableHeaderSchemaValidator.getInstance()
1410+
.validateInsertNodeMeasurements(
1411+
database,
1412+
measurementInfo,
1413+
context,
1414+
allowCreateTable,
1415+
measurementValidator,
1416+
tagColumnHandler);
1417+
}
1418+
14001419
@Override
14011420
public void validateDeviceSchema(
14021421
ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) {

0 commit comments

Comments
 (0)