Skip to content

Add independent ttl check for object file #16929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JackieTien97 merged 9 commits into from
Dec 23, 2025
Merged

Add independent ttl check for object file #16929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
86b67bd
check object ttl
shuwenwei Dec 18, 2025
4d73c0d
add ut
shuwenwei Dec 19, 2025
73fad55
fix ut
shuwenwei Dec 19, 2025
8d6ce26
add ut
shuwenwei Dec 19, 2025
f7f1465
add rate limit
shuwenwei Dec 19, 2025
3e6ce63
fix comment format
shuwenwei Dec 19, 2025
48d4901
update compaction read operation rate
shuwenwei Dec 23, 2025
c5b4507
Merge branch 'master' into object_ttl
shuwenwei Dec 23, 2025
ce9f1be
fix conflict
shuwenwei Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,7 @@ private void unbindCompactionTaskMemory(AbstractMetricService metricService) {

private Histogram settleCompactionTaskSelectionTimeCost =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
private Histogram ttlCheckForObjectFileTimeCost = DoNothingMetricManager.DO_NOTHING_HISTOGRAM;

private Histogram seqInnerSpaceCompactionTaskSelectedFileNum =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
Expand Down Expand Up @@ -766,6 +767,10 @@ public void updateCompactionTaskSelectionTimeCost(CompactionTaskType taskType, l
}
}

public void updateTTLCheckForObjectFileCost(long time) {
ttlCheckForObjectFileTimeCost.update(time);
}

public void updateCompactionTaskSelectedFileNum(
CompactionTaskType taskType, int selectedFileNum) {
switch (taskType) {
Expand Down Expand Up @@ -869,6 +874,12 @@ private void bindCompactionTaskSelection(AbstractMetricService metricService) {
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
"settle");
ttlCheckForObjectFileTimeCost =
metricService.getOrCreateHistogram(
Metric.COMPACTION_TASK_SELECTION_COST.toString(),
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
"settle-object");
seqInnerSpaceCompactionTaskSelectedFileNum =
metricService.getOrCreateHistogram(
Metric.COMPACTION_TASK_SELECTED_FILE.toString(),
Expand Down Expand Up @@ -935,7 +946,8 @@ private void bindCompactionTaskSelection(AbstractMetricService metricService) {
}

private void unbindCompactionTaskSelection(AbstractMetricService metricService) {
for (String taskType : Arrays.asList("seq", "unseq", "cross", "insertion", "settle")) {
for (String taskType :
Arrays.asList("seq", "unseq", "cross", "insertion", "settle", "settle-object")) {
metricService.remove(
MetricType.GAUGE,
Metric.COMPACTION_TASK_SELECTION.toString(),
Expand Down
15 changes: 15 additions & 0 deletions ...-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleContext;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduler;
Expand Down Expand Up @@ -3428,6 +3429,20 @@ public int executeTTLCheck() throws InterruptedException {
return trySubmitCount;
}

public void executeTTLCheckForObjectFiles() throws InterruptedException {
long startTime = System.currentTimeMillis();
List<String> allObjectDirs = TierManager.getInstance().getAllObjectFileFolders();
for (String objectDir : allObjectDirs) {
File regionObjectDir = new File(objectDir, dataRegionIdString);
if (!regionObjectDir.isDirectory()) {
continue;
}
CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionObjectDir, databaseName);
}
CompactionMetrics.getInstance()
.updateTTLCheckForObjectFileCost(System.currentTimeMillis() - startTime);
}

private boolean skipCurrentTTLAndModificationCheck() {
if (this.databaseName.equals(InformationSchema.INFORMATION_DATABASE)) {
return true;
Expand Down
6 changes: 0 additions & 6 deletions ...e/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICrossCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
Expand Down Expand Up @@ -193,11 +192,6 @@ public boolean doCompaction() {
performer.setTargetFiles(targetTsfileResourceList);
performer.setSummary(summary);
performer.perform();
if (performer instanceof ReadPointCompactionPerformer) {
for (TsFileResource resource : getAllSourceTsFiles()) {
CompactionUtils.removeDeletedObjectFiles(resource);
}
}

CompactionUtils.updateProgressIndexAndMark(
targetTsfileResourceList, selectedSequenceFiles, selectedUnsequenceFiles);
Expand Down
6 changes: 0 additions & 6 deletions ...e/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.IInnerCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
Expand Down Expand Up @@ -390,11 +389,6 @@ protected void compact(SimpleCompactionLogger compactionLogger) throws Exception
performer.setTargetFiles(filesView.targetFilesInPerformer);
performer.setSummary(summary);
performer.perform();
if (performer instanceof ReadPointCompactionPerformer) {
for (TsFileResource resource : filesView.sourceFilesInCompactionPerformer) {
CompactionUtils.removeDeletedObjectFiles(resource);
}
}

prepareTargetFiles();

Expand Down
1 change: 0 additions & 1 deletion ...pache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,6 @@ public boolean settleWithFullyDirtyFiles() throws IllegalPathException, IOExcept
if (recoverMemoryStatus) {
tsFileManager.remove(resource, resource.isSeq());
}
CompactionUtils.removeDeletedObjectFiles(resource);

boolean res = deleteTsFileOnDisk(resource);
if (res) {
Expand Down
Loading
Loading