Skip to content

Commit 92308f2

Browse files
zerolbsonyzerolbsony
authored
Only delete data, but don't delete tsfile when performing a drop column statement on the table model (#16936)
* Remove the code check port is occupied, and resolve the problem that can't rename file successfully. * Only delete data, but don't delete tsfile when performing a drop column statement on the table model or a drop tag statement on the tree model. * Fix NPE * Don't delete tsfile when use tag as a where clause in the delete statement on table model. * stash * Won't delete according tsfile files directly if IDPredicate type is not NOP when delete table data by tag. * Occur incompatible exception when merging statistics, need to rewrite statistics in current chunk metadata so that resolve "Statistics classes mismatched: class org.apache.tsfile.file.metadata.statistics.BinaryStatistics vs. class org.apache.tsfile.file.metadata.statistics.IntegerStatistics" exception. * Don't merge statistics when two types are not compatible; Fix regenerate statistics logics. * Fix out of bounds problem from unit test. * Use a constant variable instead of frequently creating a empty Binary object. * When the statement to execute is to drop a column, put it in the mods file instead of deleting the tsfile. * Repair and add the logic involve DATE data type. * Fix problem that out of bounds in the SchemaUtilsTest. * In order to avoid get chunkMetadata of other index in the array, wish to get chunkMetadata current index itself, when execute query statement. Place null value that when chunkMetaData is not compatible with chunkMetadData of target data type * Fix the issue "Only the target component should be written, not all of them". * Fix unit test. * Fix unit test.
1 parent 09db0d6 commit 92308f2

File tree

7 files changed

+516
-137
lines changed

7 files changed

+516
-137
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@
3333
import org.apache.iotdb.rpc.IoTDBConnectionException;
3434
import org.apache.iotdb.rpc.StatementExecutionException;
3535

36+
import org.apache.tsfile.enums.ColumnCategory;
37+
import org.apache.tsfile.enums.TSDataType;
3638
import org.apache.tsfile.read.common.RowRecord;
3739
import org.apache.tsfile.read.common.TimeRange;
40+
import org.apache.tsfile.write.record.Tablet;
3841
import org.awaitility.Awaitility;
3942
import org.junit.After;
4043
import org.junit.AfterClass;
@@ -60,6 +63,7 @@
6063
import java.sql.SQLException;
6164
import java.sql.Statement;
6265
import java.util.ArrayList;
66+
import java.util.Collections;
6367
import java.util.List;
6468
import java.util.Locale;
6569
import java.util.Random;
@@ -74,6 +78,7 @@
7478
import java.util.stream.Collectors;
7579
import java.util.stream.Stream;
7680

81+
import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue;
7782
import static org.junit.Assert.assertEquals;
7883
import static org.junit.Assert.assertFalse;
7984
import static org.junit.Assert.assertTrue;
@@ -2289,6 +2294,159 @@ public void testMultiDeviceCompletelyDeleteTable() throws SQLException {
22892294
cleanData(testNum);
22902295
}
22912296

2297+
@Test
2298+
public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExecutionException {
2299+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
2300+
session.executeNonQueryStatement(
2301+
"CREATE TABLE IF NOT EXISTS delete_by_tag (deviceId STRING TAG, s1 INT32 FIELD)");
2302+
2303+
session.executeNonQueryStatement(
2304+
"insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)");
2305+
session.executeNonQueryStatement(
2306+
"insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)");
2307+
session.executeNonQueryStatement(
2308+
"insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)");
2309+
session.executeNonQueryStatement(
2310+
"insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)");
2311+
2312+
session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'");
2313+
2314+
SessionDataSet dataSet =
2315+
session.executeQueryStatement("select * from delete_by_tag order by time");
2316+
assertFalse(dataSet.hasNext());
2317+
2318+
session.executeNonQueryStatement(
2319+
"insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)");
2320+
session.executeNonQueryStatement(
2321+
"insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)");
2322+
session.executeNonQueryStatement(
2323+
"insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)");
2324+
session.executeNonQueryStatement(
2325+
"insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)");
2326+
session.executeNonQueryStatement("FLUSH");
2327+
2328+
session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'");
2329+
2330+
dataSet = session.executeQueryStatement("select * from delete_by_tag order by time");
2331+
assertFalse(dataSet.hasNext());
2332+
} finally {
2333+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
2334+
session.executeNonQueryStatement("DROP TABLE IF EXISTS delete_by_tag");
2335+
}
2336+
}
2337+
}
2338+
2339+
@Test
2340+
public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException {
2341+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
2342+
session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)");
2343+
2344+
// time=1 and time=2 are INT32 and deleted by drop column
2345+
Tablet tablet =
2346+
new Tablet(
2347+
"drop_and_alter",
2348+
Collections.singletonList("s1"),
2349+
Collections.singletonList(TSDataType.INT32),
2350+
Collections.singletonList(ColumnCategory.FIELD));
2351+
tablet.addTimestamp(0, 1);
2352+
tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1));
2353+
session.insert(tablet);
2354+
tablet.reset();
2355+
2356+
session.executeNonQueryStatement("FLUSH");
2357+
2358+
tablet =
2359+
new Tablet(
2360+
"drop_and_alter",
2361+
Collections.singletonList("s1"),
2362+
Collections.singletonList(TSDataType.INT32),
2363+
Collections.singletonList(ColumnCategory.FIELD));
2364+
tablet.addTimestamp(0, 2);
2365+
tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2));
2366+
session.insert(tablet);
2367+
tablet.reset();
2368+
2369+
session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1");
2370+
2371+
// time=3 and time=4 are STRING
2372+
tablet =
2373+
new Tablet(
2374+
"drop_and_alter",
2375+
Collections.singletonList("s1"),
2376+
Collections.singletonList(TSDataType.STRING),
2377+
Collections.singletonList(ColumnCategory.FIELD));
2378+
tablet.addTimestamp(0, 3);
2379+
tablet.addValue("s1", 0, genValue(TSDataType.STRING, 3));
2380+
session.insert(tablet);
2381+
tablet.reset();
2382+
2383+
session.executeNonQueryStatement("FLUSH");
2384+
2385+
tablet =
2386+
new Tablet(
2387+
"drop_and_alter",
2388+
Collections.singletonList("s1"),
2389+
Collections.singletonList(TSDataType.STRING),
2390+
Collections.singletonList(ColumnCategory.FIELD));
2391+
tablet.addTimestamp(0, 4);
2392+
tablet.addValue("s1", 0, genValue(TSDataType.STRING, 4));
2393+
session.insert(tablet);
2394+
tablet.reset();
2395+
2396+
session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1");
2397+
session.executeNonQueryStatement("ALTER TABLE drop_and_alter ADD COLUMN s1 TEXT");
2398+
2399+
// time=5 and time=6 are TEXT
2400+
tablet =
2401+
new Tablet(
2402+
"drop_and_alter",
2403+
Collections.singletonList("s1"),
2404+
Collections.singletonList(TSDataType.TEXT),
2405+
Collections.singletonList(ColumnCategory.FIELD));
2406+
tablet.addTimestamp(0, 5);
2407+
tablet.addValue("s1", 0, genValue(TSDataType.STRING, 5));
2408+
session.insert(tablet);
2409+
tablet.reset();
2410+
2411+
session.executeNonQueryStatement("FLUSH");
2412+
2413+
tablet =
2414+
new Tablet(
2415+
"drop_and_alter",
2416+
Collections.singletonList("s1"),
2417+
Collections.singletonList(TSDataType.TEXT),
2418+
Collections.singletonList(ColumnCategory.FIELD));
2419+
tablet.addTimestamp(0, 6);
2420+
tablet.addValue("s1", 0, genValue(TSDataType.STRING, 6));
2421+
session.insert(tablet);
2422+
tablet.reset();
2423+
2424+
SessionDataSet dataSet =
2425+
session.executeQueryStatement("select * from drop_and_alter order by time");
2426+
// s1 is dropped but the time should remain
2427+
RowRecord rec;
2428+
int cnt = 0;
2429+
for (int i = 1; i < 7; i++) {
2430+
rec = dataSet.next();
2431+
assertEquals(i, rec.getFields().get(0).getLongV());
2432+
LOGGER.error(
2433+
"time is {}, value is {}, value type is {}",
2434+
rec.getFields().get(0).getLongV(),
2435+
rec.getFields().get(1),
2436+
rec.getFields().get(1).getDataType());
2437+
// assertNull(rec.getFields().get(1).getDataType());
2438+
// Assert.assertEquals(TSDataType.TEXT, rec.getFields().get(1).getDataType());
2439+
cnt++;
2440+
}
2441+
Assert.assertEquals(6, cnt);
2442+
assertFalse(dataSet.hasNext());
2443+
} finally {
2444+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
2445+
session.executeNonQueryStatement("DROP TABLE IF EXISTS drop_and_alter");
2446+
}
2447+
}
2448+
}
2449+
22922450
private static void prepareDatabase() {
22932451
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
22942452
Statement statement = connection.createStatement()) {

‎integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ private void doWriteAndAlter(TSDataType from, TSDataType to)
140140
throws IoTDBConnectionException, StatementExecutionException {
141141
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) {
142142
session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'");
143+
session.executeNonQueryStatement("SET CONFIGURATION enable_seq_space_compaction='false'");
143144
if (from == TSDataType.DATE && !to.isCompatible(from)) {
144145
throw new NotSupportedException("Not supported DATE type.");
145146
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -268,19 +268,19 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata(
268268
modified = (modified || alignedChunkMetadata.isModified());
269269
TSDataType targetDataType = alignedFullPath.getSchemaList().get(index).getType();
270270
if (targetDataType.equals(TSDataType.STRING)
271-
&& (alignedChunkMetadata.getValueChunkMetadataList().stream()
272-
.filter(iChunkMetadata -> iChunkMetadata.getDataType() != targetDataType)
273-
.count()
274-
> 0)) {
271+
&& ((alignedChunkMetadata.getValueChunkMetadataList().get(index) != null)
272+
&& (alignedChunkMetadata.getValueChunkMetadataList().get(index).getDataType()
273+
!= targetDataType))) {
275274
// create new statistics object via new data type, and merge statistics information
276-
alignedChunkMetadata =
277-
SchemaUtils.rewriteAlignedChunkMetadataStatistics(alignedChunkMetadata, targetDataType);
275+
SchemaUtils.rewriteAlignedChunkMetadataStatistics(
276+
alignedChunkMetadata, index, targetDataType);
278277
alignedChunkMetadata.setModified(true);
279278
}
280279
if (!useFakeStatistics) {
281280
timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
282281
for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
283-
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
282+
if (!alignedChunkMetadata.getValueChunkMetadataList().isEmpty()
283+
&& alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
284284
exist[i] = true;
285285
valueTimeSeriesMetadataList
286286
.get(i)
@@ -542,8 +542,18 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata(
542542
boolean isModified = false;
543543
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
544544
isModified = (isModified || chunkMetadata.isModified());
545+
TSDataType targetDataType = fullPath.getMeasurementSchema().getType();
546+
if (targetDataType.equals(TSDataType.STRING)
547+
&& (chunkMetadata.getDataType() != targetDataType)) {
548+
// create new statistics object via new data type, and merge statistics information
549+
SchemaUtils.rewriteNonAlignedChunkMetadataStatistics(
550+
(ChunkMetadata) chunkMetadata, targetDataType);
551+
chunkMetadata.setModified(true);
552+
}
545553
if (!useFakeStatistics) {
546-
seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
554+
if (chunkMetadata != null && targetDataType.isCompatible(chunkMetadata.getDataType())) {
555+
seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
556+
}
547557
continue;
548558
}
549559
startTime = Math.min(startTime, chunkMetadata.getStartTime());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@
116116
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
117117
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
118118
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessorInfo;
119+
import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate;
119120
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
120121
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
121122
import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
@@ -3235,18 +3236,14 @@ private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, M
32353236
List<TsFileResource> deletedByMods = new ArrayList<>();
32363237
List<TsFileResource> deletedByFiles = new ArrayList<>();
32373238
boolean isDropMeasurementExist = false;
3238-
boolean isDropTagExist = false;
3239+
IDPredicate.IDPredicateType idPredicateType = null;
32393240

32403241
if (deletion instanceof TableDeletionEntry) {
3241-
TableDeletionEntry entry = (TableDeletionEntry) deletion;
3242-
isDropMeasurementExist = !entry.getPredicate().getMeasurementNames().isEmpty();
3243-
} else {
3244-
TreeDeletionEntry entry = (TreeDeletionEntry) deletion;
3245-
if (entry.getPathPattern() instanceof MeasurementPath) {
3246-
Map<String, String> tagMap = ((MeasurementPath) entry.getPathPattern()).getTagMap();
3247-
isDropTagExist = (tagMap != null) && !tagMap.isEmpty();
3248-
}
3242+
TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion;
3243+
isDropMeasurementExist = !tableDeletionEntry.getPredicate().getMeasurementNames().isEmpty();
3244+
idPredicateType = tableDeletionEntry.getPredicate().getIdPredicateType();
32493245
}
3246+
32503247
for (TsFileResource sealedTsFile : sealedTsFiles) {
32513248
if (canSkipDelete(sealedTsFile, deletion)) {
32523249
continue;
@@ -3310,7 +3307,9 @@ private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, M
33103307
fileStartTime,
33113308
fileEndTime);
33123309
}
3313-
if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)) {
3310+
if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)
3311+
&& idPredicateType.equals(IDPredicate.IDPredicateType.NOP)
3312+
&& !isDropMeasurementExist) {
33143313
++matchSize;
33153314
} else {
33163315
deletedByMods.add(sealedTsFile);
@@ -3343,7 +3342,7 @@ private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, M
33433342
} // else do nothing
33443343
}
33453344

3346-
if (!deletedByFiles.isEmpty() && !isDropMeasurementExist && !isDropTagExist) {
3345+
if (!deletedByFiles.isEmpty()) {
33473346
deleteTsFileCompletely(deletedByFiles);
33483347
if (logger.isDebugEnabled()) {
33493348
logger.debug(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ public IDPredicate getIdPredicate() {
7676
return idPredicate;
7777
}
7878

79+
public IDPredicate.IDPredicateType getIdPredicateType() {
80+
return this.idPredicate.type;
81+
}
82+
7983
public String getTableName() {
8084
return tableName;
8185
}

0 commit comments

Comments
 (0)