Skip to content

Commit 82a1dc3

Browse files
authored
Fixed the bug that set table properties ttl='inf' cannot be parsed && ttl='INF' does not take effect && set to default may generate NPE on dataNodes && some minor bugs of delete devices && Enable "inf" in databaseSchema / show ttl for databases in table model / table default ttl = database ttl (apache#14147)
1 parent 4a76dfb commit 82a1dc3

File tree

14 files changed

+150
-76
lines changed

14 files changed

+150
-76
lines changed

integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,7 @@ public void testIoTDBPatternWithDataBaseAndTable() throws Exception {
564564
TestUtils.assertDataEventuallyOnEnv(
565565
receiverEnv,
566566
"show databases",
567-
"Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
567+
"Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
568568
Collections.emptySet(),
569569
null);
570570
}

integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public void testManageDatabase() {
6767
final Statement statement = connection.createStatement()) {
6868

6969
// create
70-
statement.execute("create database test");
70+
statement.execute("create database test with (ttl='INF')");
7171

7272
// create duplicated database without IF NOT EXISTS
7373
try {
@@ -81,6 +81,7 @@ public void testManageDatabase() {
8181
statement.execute("create database IF NOT EXISTS test");
8282

8383
String[] databaseNames = new String[] {"test"};
84+
String[] TTLs = new String[] {"INF"};
8485
int[] schemaReplicaFactors = new int[] {1};
8586
int[] dataReplicaFactors = new int[] {1};
8687
int[] timePartitionInterval = new int[] {604800000};
@@ -96,9 +97,10 @@ public void testManageDatabase() {
9697
}
9798
while (resultSet.next()) {
9899
assertEquals(databaseNames[cnt], resultSet.getString(1));
99-
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(2));
100-
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(3));
101-
assertEquals(timePartitionInterval[cnt], resultSet.getLong(4));
100+
assertEquals(TTLs[cnt], resultSet.getString(2));
101+
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3));
102+
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4));
103+
assertEquals(timePartitionInterval[cnt], resultSet.getLong(5));
102104
cnt++;
103105
}
104106
assertEquals(databaseNames.length, cnt);
@@ -115,10 +117,11 @@ public void testManageDatabase() {
115117
}
116118
while (resultSet.next()) {
117119
assertEquals(databaseNames[cnt], resultSet.getString(1));
118-
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(2));
119-
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(3));
120-
assertEquals(timePartitionInterval[cnt], resultSet.getLong(4));
121-
assertEquals(model[cnt], resultSet.getString(5));
120+
assertEquals(TTLs[cnt], resultSet.getString(2));
121+
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3));
122+
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4));
123+
assertEquals(timePartitionInterval[cnt], resultSet.getLong(5));
124+
assertEquals(model[cnt], resultSet.getString(6));
122125
cnt++;
123126
}
124127
assertEquals(databaseNames.length, cnt);
@@ -154,9 +157,9 @@ public void testManageDatabase() {
154157

155158
// Test create database with properties
156159
statement.execute(
157-
"create database test_prop with (schema_replication_factor=DEFAULT, data_replication_factor=3, time_partition_interval=100000)");
158-
160+
"create database test_prop with (ttl=300, schema_replication_factor=DEFAULT, data_replication_factor=3, time_partition_interval=100000)");
159161
databaseNames = new String[] {"test_prop"};
162+
TTLs = new String[] {"300"};
160163
dataReplicaFactors = new int[] {3};
161164
timePartitionInterval = new int[] {100000};
162165

@@ -170,9 +173,10 @@ public void testManageDatabase() {
170173
}
171174
while (resultSet.next()) {
172175
assertEquals(databaseNames[cnt], resultSet.getString(1));
173-
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(2));
174-
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(3));
175-
assertEquals(timePartitionInterval[cnt], resultSet.getLong(4));
176+
assertEquals(TTLs[cnt], resultSet.getString(2));
177+
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(3));
178+
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(4));
179+
assertEquals(timePartitionInterval[cnt], resultSet.getLong(5));
176180
cnt++;
177181
}
178182
assertEquals(databaseNames.length, cnt);

integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public void testManageTable() {
6868
final Statement statement = connection.createStatement()) {
6969

7070
statement.execute("create database test1");
71-
statement.execute("create database test2");
71+
statement.execute("create database test2 with (ttl=300)");
7272

7373
// should specify database before create table
7474
try {
@@ -287,8 +287,11 @@ public void testManageTable() {
287287

288288
statement.execute("alter table if exists table3 add column speed DOUBLE MEASUREMENT");
289289

290-
tableNames = new String[] {"table2"};
291-
ttls = new String[] {"6600000"};
290+
// Test create table with only time column
291+
statement.execute("create table table3()");
292+
293+
tableNames = new String[] {"table3", "table2"};
294+
ttls = new String[] {"300", "6600000"};
292295

293296
// show tables from current database
294297
try (final ResultSet resultSet = statement.executeQuery("SHOW tables")) {
@@ -307,8 +310,25 @@ public void testManageTable() {
307310
assertEquals(tableNames.length, cnt);
308311
}
309312

310-
// Test create table with only time column
311-
statement.execute("create table table3()");
313+
statement.execute("alter table table3 set properties ttl=300");
314+
statement.execute("alter table table3 set properties ttl=DEFAULT");
315+
316+
// The table3's ttl shall be also 300
317+
try (final ResultSet resultSet = statement.executeQuery("SHOW tables")) {
318+
int cnt = 0;
319+
ResultSetMetaData metaData = resultSet.getMetaData();
320+
assertEquals(showTablesColumnHeaders.size(), metaData.getColumnCount());
321+
for (int i = 0; i < showTablesColumnHeaders.size(); i++) {
322+
assertEquals(
323+
showTablesColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1));
324+
}
325+
while (resultSet.next()) {
326+
assertEquals(tableNames[cnt], resultSet.getString(1));
327+
assertEquals(ttls[cnt], resultSet.getString(2));
328+
cnt++;
329+
}
330+
assertEquals(tableNames.length, cnt);
331+
}
312332

313333
// show tables from a non-exist database
314334
try {

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,7 @@ public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) {
360360
final String database = databaseSchema.getName();
361361
final TDatabaseInfo databaseInfo = new TDatabaseInfo();
362362
databaseInfo.setName(database);
363+
databaseInfo.setTTL(databaseSchema.isSetTTL() ? databaseSchema.getTTL() : Long.MAX_VALUE);
363364
databaseInfo.setSchemaReplicationFactor(databaseSchema.getSchemaReplicationFactor());
364365
databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor());
365366
databaseInfo.setTimePartitionOrigin(databaseSchema.getTimePartitionOrigin());
@@ -1321,7 +1322,11 @@ public synchronized Pair<TSStatus, TsTable> updateTableProperties(
13211322
updatedProperties.forEach(
13221323
(k, v) -> {
13231324
originalProperties.put(k, originalTable.getPropValue(k).orElse(null));
1324-
updatedTable.addProp(k, v);
1325+
if (Objects.nonNull(v)) {
1326+
updatedTable.addProp(k, v);
1327+
} else {
1328+
updatedTable.removeProp(k);
1329+
}
13251330
});
13261331

13271332
return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable);

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,6 @@
101101
import java.util.HashMap;
102102
import java.util.HashSet;
103103
import java.util.List;
104-
import java.util.Locale;
105104
import java.util.Map;
106105
import java.util.Optional;
107106
import java.util.Set;
@@ -1125,9 +1124,7 @@ public ShowTableResp showTables(final ShowTablePlan plan) {
11251124
final TTableInfo info =
11261125
new TTableInfo(
11271126
pair.getLeft().getTableName(),
1128-
pair.getLeft()
1129-
.getPropValue(TTL_PROPERTY.toLowerCase(Locale.ENGLISH))
1130-
.orElse(TTL_INFINITE));
1127+
pair.getLeft().getPropValue(TTL_PROPERTY).orElse(TTL_INFINITE));
11311128
info.setState(pair.getRight().ordinal());
11321129
return info;
11331130
})
@@ -1140,9 +1137,7 @@ public ShowTableResp showTables(final ShowTablePlan plan) {
11401137
tsTable ->
11411138
new TTableInfo(
11421139
tsTable.getTableName(),
1143-
tsTable
1144-
.getPropValue(TTL_PROPERTY.toLowerCase(Locale.ENGLISH))
1145-
.orElse(TTL_INFINITE)))
1140+
tsTable.getPropValue(TTL_PROPERTY).orElse(TTL_INFINITE)))
11461141
.collect(Collectors.toList()));
11471142
} catch (final MetadataException e) {
11481143
return new ShowTableResp(

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -816,11 +816,19 @@ public void renameTableColumn(
816816
public void setTableProperties(
817817
final PartialPath database, final String tableName, final Map<String, String> tableProperties)
818818
throws MetadataException {
819-
final TsTable table = getTable(database, tableName);
819+
final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode();
820+
if (!databaseNode.hasChild(tableName)) {
821+
throw new TableNotExistsException(
822+
database.getFullPath().substring(ROOT.length() + 1), tableName);
823+
}
824+
final TsTable table = ((ConfigTableNode) databaseNode.getChild(tableName)).getTable();
820825
tableProperties.forEach(
821826
(k, v) -> {
822827
if (Objects.nonNull(v)) {
823828
table.addProp(k, v);
829+
} else if (k.equals(TsTable.TTL_PROPERTY)
830+
&& databaseNode.getDatabaseSchema().isSetTTL()) {
831+
table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL()));
824832
} else {
825833
table.removeProp(k);
826834
}

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan;
3333
import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan;
3434
import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan;
35+
import org.apache.iotdb.confignode.exception.DatabaseNotExistsException;
3536
import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
3637
import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
3738
import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException;
@@ -41,6 +42,7 @@
4142
import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils;
4243
import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState;
4344
import org.apache.iotdb.confignode.procedure.store.ProcedureType;
45+
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
4446
import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq;
4547
import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp;
4648
import org.apache.iotdb.rpc.TSStatusCode;
@@ -141,9 +143,14 @@ private void checkTableExistence(final ConfigNodeProcedureEnv env) {
141143
database.substring(ROOT.length() + 1), table.getTableName()),
142144
TABLE_ALREADY_EXISTS.getStatusCode())));
143145
} else {
146+
final TDatabaseSchema schema =
147+
env.getConfigManager().getClusterSchemaManager().getDatabaseSchemaByName(database);
148+
if (schema.isSetTTL() && !table.getPropValue(TsTable.TTL_PROPERTY).isPresent()) {
149+
table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL()));
150+
}
144151
setNextState(CreateTableState.PRE_CREATE);
145152
}
146-
} catch (final MetadataException e) {
153+
} catch (final MetadataException | DatabaseNotExistsException e) {
147154
setFailure(new ProcedureException(e));
148155
}
149156
}

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DeleteDevicesProcedure.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,12 @@
6161

6262
import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
6363
import static org.apache.iotdb.commons.schema.SchemaConstant.ROOT;
64+
import static org.apache.iotdb.confignode.procedure.state.schema.DeleteDevicesState.CHECK_TABLE_EXISTENCE;
6465
import static org.apache.iotdb.confignode.procedure.state.schema.DeleteDevicesState.CLEAN_DATANODE_SCHEMA_CACHE;
6566
import static org.apache.iotdb.confignode.procedure.state.schema.DeleteDevicesState.CONSTRUCT_BLACK_LIST;
6667
import static org.apache.iotdb.confignode.procedure.state.schema.DeleteDevicesState.DELETE_DATA;
6768
import static org.apache.iotdb.confignode.procedure.state.schema.DeleteDevicesState.DELETE_DEVICE_SCHEMA;
68-
import static org.apache.iotdb.rpc.TSStatusCode.TABLE_ALREADY_EXISTS;
69+
import static org.apache.iotdb.rpc.TSStatusCode.TABLE_NOT_EXISTS;
6970

7071
public class DeleteDevicesProcedure extends AbstractAlterOrDropTableProcedure<DeleteDevicesState> {
7172
private static final Logger LOGGER = LoggerFactory.getLogger(DeleteDevicesProcedure.class);
@@ -103,7 +104,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DeleteDe
103104
try {
104105
switch (state) {
105106
case CHECK_TABLE_EXISTENCE:
106-
LOGGER.info("Check the existence of table {}.{}", database, table.getTableName());
107+
LOGGER.info("Check the existence of table {}.{}", database, tableName);
107108
checkTableExistence(env);
108109
break;
109110
case CONSTRUCT_BLACK_LIST:
@@ -133,24 +134,23 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DeleteDe
133134
}
134135
return Flow.HAS_MORE_STATE;
135136
} finally {
136-
LOGGER.info(
137-
"DeleteTimeSeries-[{}] costs {}ms", state, (System.currentTimeMillis() - startTime));
137+
LOGGER.info("DeleteDevices-[{}] costs {}ms", state, (System.currentTimeMillis() - startTime));
138138
}
139139
}
140140

141141
private void checkTableExistence(final ConfigNodeProcedureEnv env) {
142142
try {
143-
if (env.getConfigManager()
143+
if (!env.getConfigManager()
144144
.getClusterSchemaManager()
145-
.getTableIfExists(database, table.getTableName())
145+
.getTableIfExists(database, tableName)
146146
.isPresent()) {
147147
setFailure(
148148
new ProcedureException(
149149
new IoTDBException(
150150
String.format(
151-
"Table '%s.%s' already exists.",
152-
database.substring(ROOT.length() + 1), table.getTableName()),
153-
TABLE_ALREADY_EXISTS.getStatusCode())));
151+
"Table '%s.%s' not exists.",
152+
database.substring(ROOT.length() + 1), tableName),
153+
TABLE_NOT_EXISTS.getStatusCode())));
154154
} else {
155155
setNextState(CONSTRUCT_BLACK_LIST);
156156
}
@@ -328,7 +328,7 @@ protected int getStateId(final DeleteDevicesState deleteDevicesState) {
328328

329329
@Override
330330
protected DeleteDevicesState getInitialState() {
331-
return CONSTRUCT_BLACK_LIST;
331+
return CHECK_TABLE_EXISTENCE;
332332
}
333333

334334
@Override

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,13 +545,15 @@ private ColumnHeaderConstant() {
545545
public static final List<ColumnHeader> showDBColumnHeaders =
546546
ImmutableList.of(
547547
new ColumnHeader(DATABASE, TSDataType.TEXT),
548+
new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
548549
new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
549550
new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
550551
new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64));
551552

552553
public static final List<ColumnHeader> showDBDetailsColumnHeaders =
553554
ImmutableList.of(
554555
new ColumnHeader(DATABASE, TSDataType.TEXT),
556+
new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
555557
new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
556558
new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
557559
new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64),

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,9 @@
128128
import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
129129
import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement;
130130

131+
import org.apache.tsfile.common.conf.TSFileConfig;
131132
import org.apache.tsfile.enums.TSDataType;
133+
import org.apache.tsfile.utils.Binary;
132134
import org.apache.tsfile.utils.Pair;
133135

134136
import java.util.Collections;
@@ -137,9 +139,11 @@
137139
import java.util.Locale;
138140
import java.util.Map;
139141
import java.util.Objects;
142+
import java.util.Optional;
140143

141144
import static org.apache.iotdb.commons.conf.IoTDBConstant.MAX_DATABASE_NAME_LENGTH;
142-
import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES_2_DEFAULT_VALUE_MAP;
145+
import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE;
146+
import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES;
143147
import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY;
144148
import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.DATA_REGION_GROUP_NUM_KEY;
145149
import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.DATA_REPLICATION_FACTOR_KEY;
@@ -205,6 +209,14 @@ protected IConfigTask visitCreateDB(final CreateDB node, final MPPQueryContext c
205209

206210
switch (key) {
207211
case TTL_KEY:
212+
final Optional<String> strValue = parseStringFromLiteralIfBinary(value);
213+
if (strValue.isPresent()) {
214+
if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) {
215+
throw new SemanticException(
216+
"ttl value must be 'INF' or a long literal, but now is: " + value);
217+
}
218+
break;
219+
}
208220
schema.setTTL(parseLongFromLiteral(value, TTL_KEY));
209221
break;
210222
case SCHEMA_REPLICATION_FACTOR_KEY:
@@ -443,21 +455,20 @@ private Map<String, String> convertPropertiesToMap(
443455
final Map<String, String> map = new HashMap<>();
444456
for (final Property property : propertyList) {
445457
final String key = property.getName().getValue().toLowerCase(Locale.ENGLISH);
446-
if (TABLE_ALLOWED_PROPERTIES_2_DEFAULT_VALUE_MAP.containsKey(key)) {
458+
if (TABLE_ALLOWED_PROPERTIES.contains(key)) {
447459
if (!property.isSetToDefault()) {
448460
final Expression value = property.getNonDefaultValue();
449-
if (value instanceof Literal
450-
&& Objects.equals(
451-
((Literal) value).getTsValue(),
452-
TABLE_ALLOWED_PROPERTIES_2_DEFAULT_VALUE_MAP.get(key))) {
453-
// Ignore default values
461+
final Optional<String> strValue = parseStringFromLiteralIfBinary(value);
462+
if (strValue.isPresent()) {
463+
if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) {
464+
throw new SemanticException(
465+
"ttl value must be 'INF' or a long literal, but now is: " + value);
466+
}
467+
map.put(key, strValue.get());
454468
continue;
455469
}
456470
// TODO: support validation for other properties
457-
map.put(
458-
key,
459-
String.valueOf(
460-
parseLongFromLiteral(value, TTL_PROPERTY.toLowerCase(Locale.ENGLISH))));
471+
map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY)));
461472
} else if (serializeDefault) {
462473
map.put(key, null);
463474
}
@@ -542,6 +553,13 @@ protected IConfigTask visitSetConfiguration(SetConfiguration node, MPPQueryConte
542553
return new SetConfigurationTask(((SetConfigurationStatement) node.getInnerTreeStatement()));
543554
}
544555

556+
private Optional<String> parseStringFromLiteralIfBinary(final Object value) {
557+
return value instanceof Literal && ((Literal) value).getTsValue() instanceof Binary
558+
? Optional.of(
559+
((Binary) ((Literal) value).getTsValue()).getStringValue(TSFileConfig.STRING_CHARSET))
560+
: Optional.empty();
561+
}
562+
545563
private long parseLongFromLiteral(final Object value, final String name) {
546564
if (!(value instanceof LongLiteral)) {
547565
throw new SemanticException(

0 commit comments

Comments
 (0)