Skip to content

Commit 4d73c0d

Browse files
shuwenweishuwenwei
committed
add ut
1 parent 86b67bd commit 4d73c0d

File tree

3 files changed

+97
-9
lines changed

3 files changed

+97
-9
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -539,7 +539,7 @@ public static void executeTTLCheckObjectFilesForTableModel(
539539
continue;
540540
}
541541
// buffer 60s to avoid concurrent issues with querying
542-
final long timeLowerBoundInMS = CommonDateTimeUtils.currentTime() - ttlInMS + 60 * 1000;
542+
final long timeLowerBoundInMS = CommonDateTimeUtils.currentTime() - ttlInMS - 60 * 1000;
543543
try {
544544
recursiveTTLCheckForTableDir(
545545
tableDir, 0, tsTable.getTagNum() + 1, !restrictObjectLimit, timeLowerBoundInMS);
@@ -554,6 +554,8 @@ public static void executeTTLCheckObjectFilesForTableModel(
554554
}
555555
}
556556

557+
// We try to avoid expensive 'stat' system calls by first checking file name and only performing
558+
// Files.readAttributes when the file may be expired
557559
private static void recursiveTTLCheckForTableDir(
558560
File currentFile,
559561
int depth,

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java‎

Lines changed: 93 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,13 @@
3939
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
4040
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
4141
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask;
42+
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
4243
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
4344
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
4445
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
4546
import org.apache.iotdb.db.utils.ObjectTypeUtils;
4647

48+
import com.google.common.io.BaseEncoding;
4749
import org.apache.tsfile.enums.ColumnCategory;
4850
import org.apache.tsfile.enums.TSDataType;
4951
import org.apache.tsfile.exception.write.WriteProcessException;
@@ -67,8 +69,8 @@
6769
import java.io.File;
6870
import java.io.IOException;
6971
import java.nio.ByteBuffer;
72+
import java.nio.charset.StandardCharsets;
7073
import java.nio.file.Files;
71-
import java.nio.file.Path;
7274
import java.util.Arrays;
7375
import java.util.Collections;
7476

@@ -148,6 +150,10 @@ public void testSeqCompactionWithTTL() throws IOException, WriteProcessException
148150
new ReadChunkCompactionPerformer(),
149151
0);
150152
Assert.assertTrue(task.start());
153+
154+
Assert.assertTrue(pair1.getRight().exists());
155+
Assert.assertTrue(pair2.getRight().exists());
156+
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG);
151157
Assert.assertFalse(pair1.getRight().exists());
152158
Assert.assertTrue(pair2.getRight().exists());
153159
}
@@ -169,6 +175,10 @@ public void testUnseqCompactionWithTTL() throws IOException, WriteProcessExcepti
169175
new FastCompactionPerformer(false),
170176
0);
171177
Assert.assertTrue(task.start());
178+
179+
Assert.assertTrue(pair1.getRight().exists());
180+
Assert.assertTrue(pair2.getRight().exists());
181+
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG);
172182
Assert.assertFalse(pair2.getRight().exists());
173183
Assert.assertTrue(pair1.getRight().exists());
174184
}
@@ -191,6 +201,9 @@ public void testUnseqCompactionWithReadPointWithTTL() throws IOException, WriteP
191201
0);
192202
Assert.assertTrue(task.start());
193203
Assert.assertTrue(pair1.getRight().exists());
204+
Assert.assertTrue(pair2.getRight().exists());
205+
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG);
206+
Assert.assertTrue(pair1.getRight().exists());
194207
Assert.assertFalse(pair2.getRight().exists());
195208
}
196209

@@ -212,6 +225,10 @@ public void testCrossCompactionWithTTL() throws IOException, WriteProcessExcepti
212225
1,
213226
0);
214227
Assert.assertTrue(task.start());
228+
229+
Assert.assertTrue(pair1.getRight().exists());
230+
Assert.assertTrue(pair2.getRight().exists());
231+
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG);
215232
Assert.assertFalse(pair2.getRight().exists());
216233
Assert.assertTrue(pair1.getRight().exists());
217234
}
@@ -234,10 +251,36 @@ public void testSettleCompaction() throws IOException, WriteProcessException {
234251
new FastCompactionPerformer(true),
235252
0);
236253
Assert.assertTrue(task.start());
254+
255+
Assert.assertTrue(pair1.getRight().exists());
256+
Assert.assertTrue(pair2.getRight().exists());
257+
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG);
237258
Assert.assertFalse(pair1.getRight().exists());
238259
Assert.assertTrue(pair2.getRight().exists());
239260
}
240261

262+
@Test
263+
public void testTTLCheck() throws IOException {
264+
config.setRestrictObjectLimit(false);
265+
try {
266+
File file1 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() + 100000, false);
267+
File file2 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() + 200000, true);
268+
File file3 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() - 100000, true);
269+
File file4 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() - 200000, false);
270+
Assert.assertTrue(file1.exists());
271+
Assert.assertTrue(file2.exists());
272+
Assert.assertTrue(file3.exists());
273+
Assert.assertTrue(file4.exists());
274+
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG);
275+
Assert.assertTrue(file1.exists());
276+
Assert.assertTrue(file2.exists());
277+
Assert.assertFalse(file3.exists());
278+
Assert.assertFalse(file4.exists());
279+
} finally {
280+
config.setRestrictObjectLimit(true);
281+
}
282+
}
283+
241284
@Test
242285
public void testPlainObjectBinaryReplaceRegionId() {
243286
IObjectPath objectPath = new PlainObjectPath(1, 0, new StringArrayDeviceID("t1.d1"), "s1");
@@ -280,13 +323,32 @@ public void testBase32ObjectBinaryReplaceRegionId() {
280323
private Pair<TsFileResource, File> generateTsFileAndObject(
281324
boolean seq, long timestamp, int regionIdInTsFile) throws IOException, WriteProcessException {
282325
TsFileResource resource = createEmptyFileAndResource(seq);
283-
Path testFile1 = Files.createTempFile(regionDir.toPath(), "test_", ".bin");
326+
File dir =
327+
new File(
328+
regionDir.getPath()
329+
+ File.separator
330+
+ "t1"
331+
+ File.separator
332+
+ "d1"
333+
+ File.separator
334+
+ "s1");
335+
dir.mkdirs();
336+
File testFile1 = new File(dir, timestamp + ".bin");
284337
byte[] content = new byte[100];
285338
for (int i = 0; i < 100; i++) {
286339
content[i] = (byte) i;
287340
}
288-
Files.write(testFile1, content);
289-
String relativePathInTsFile = regionIdInTsFile + File.separator + testFile1.toFile().getName();
341+
Files.write(testFile1.toPath(), content);
342+
String relativePathInTsFile =
343+
regionIdInTsFile
344+
+ File.separator
345+
+ "t1"
346+
+ File.separator
347+
+ "d1"
348+
+ File.separator
349+
+ "s1"
350+
+ File.separator
351+
+ testFile1.getName();
290352
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + relativePathInTsFile.length());
291353
buffer.putLong(100L);
292354
buffer.put(BytesUtils.stringToBytes(relativePathInTsFile));
@@ -296,7 +358,8 @@ private Pair<TsFileResource, File> generateTsFileAndObject(
296358
writer.getSchema().registerTableSchema(tableSchema);
297359
writer.startChunkGroup(deviceID);
298360
AlignedChunkWriterImpl alignedChunkWriter =
299-
new AlignedChunkWriterImpl(Arrays.asList(new MeasurementSchema("s1", TSDataType.OBJECT)));
361+
new AlignedChunkWriterImpl(
362+
Collections.singletonList(new MeasurementSchema("s1", TSDataType.OBJECT)));
300363
alignedChunkWriter.write(timestamp);
301364
alignedChunkWriter.write(timestamp, new Binary(buffer.array()), false);
302365
alignedChunkWriter.sealCurrentPage();
@@ -309,6 +372,30 @@ private Pair<TsFileResource, File> generateTsFileAndObject(
309372
resource.serialize();
310373
resource.deserialize();
311374
resource.setStatus(TsFileResourceStatus.NORMAL);
312-
return new Pair<>(resource, testFile1.toFile());
375+
return new Pair<>(resource, testFile1);
376+
}
377+
378+
private File generateBase32ObjectFile(File regionDir, long timestamp, boolean internalLevel)
379+
throws IOException {
380+
File dir =
381+
new File(
382+
regionDir.getPath()
383+
+ File.separator
384+
+ toBase32Str("t1")
385+
+ (internalLevel ? "" : (File.separator + toBase32Str("d1")))
386+
+ File.separator
387+
+ toBase32Str("s1"));
388+
dir.mkdirs();
389+
File testFile1 = new File(dir, timestamp + ".bin");
390+
byte[] content = new byte[100];
391+
for (int i = 0; i < 100; i++) {
392+
content[i] = (byte) i;
393+
}
394+
Files.write(testFile1.toPath(), content);
395+
return testFile1;
396+
}
397+
398+
private String toBase32Str(String str) {
399+
return BaseEncoding.base32().omitPadding().encode(str.getBytes(StandardCharsets.UTF_8));
313400
}
314401
}

‎iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,7 @@ public class CommonConfig {
142142
private int ttlRuleCapacity = 1000;
143143

144144
/** The interval of ttl check task in each database. The unit is ms. Default is 2 hours. */
145-
// private long ttlCheckInterval = 7_200_000L;
146-
private long ttlCheckInterval = 10_000L;
145+
private long ttlCheckInterval = 7_200_000L;
147146

148147
/** Thrift socket and connection timeout between data node and config node. */
149148
private int cnConnectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(60);

0 commit comments

Comments
 (0)