Add independent ttl check for object file (#16929)

Author: shuwenwei

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java

@@ -703,6 +703,7 @@ private void unbindCompactionTaskMemory(AbstractMetricService metricService) {
 
   private Histogram settleCompactionTaskSelectionTimeCost =
       DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
+  private Histogram ttlCheckForObjectFileTimeCost = DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
 
   private Histogram seqInnerSpaceCompactionTaskSelectedFileNum =
       DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
@@ -766,6 +767,10 @@ public void updateCompactionTaskSelectionTimeCost(CompactionTaskType taskType, l
     }
   }
 
+  public void updateTTLCheckForObjectFileCost(long time) {
+    ttlCheckForObjectFileTimeCost.update(time);
+  }
+
   public void updateCompactionTaskSelectedFileNum(
       CompactionTaskType taskType, int selectedFileNum) {
     switch (taskType) {
@@ -869,6 +874,12 @@ private void bindCompactionTaskSelection(AbstractMetricService metricService) {
             MetricLevel.IMPORTANT,
             Tag.NAME.toString(),
             "settle");
+    ttlCheckForObjectFileTimeCost =
+        metricService.getOrCreateHistogram(
+            Metric.COMPACTION_TASK_SELECTION_COST.toString(),
+            MetricLevel.IMPORTANT,
+            Tag.NAME.toString(),
+            "settle-object");
     seqInnerSpaceCompactionTaskSelectedFileNum =
         metricService.getOrCreateHistogram(
             Metric.COMPACTION_TASK_SELECTED_FILE.toString(),
@@ -935,7 +946,8 @@ private void bindCompactionTaskSelection(AbstractMetricService metricService) {
   }
 
   private void unbindCompactionTaskSelection(AbstractMetricService metricService) {
-    for (String taskType : Arrays.asList("seq", "unseq", "cross", "insertion", "settle")) {
+    for (String taskType :
+        Arrays.asList("seq", "unseq", "cross", "insertion", "settle", "settle-object")) {
       metricService.remove(
           MetricType.GAUGE,
           Metric.COMPACTION_TASK_SELECTION.toString(),

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

@@ -103,6 +103,7 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverManager;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleContext;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduler;
@@ -3428,6 +3429,20 @@ public int executeTTLCheck() throws InterruptedException {
     return trySubmitCount;
   }
 
+  public void executeTTLCheckForObjectFiles() throws InterruptedException {
+    long startTime = System.currentTimeMillis();
+    List<String> allObjectDirs = TierManager.getInstance().getAllObjectFileFolders();
+    for (String objectDir : allObjectDirs) {
+      File regionObjectDir = new File(objectDir, dataRegionIdString);
+      if (!regionObjectDir.isDirectory()) {
+        continue;
+      }
+      CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionObjectDir, databaseName);
+    }
+    CompactionMetrics.getInstance()
+        .updateTTLCheckForObjectFileCost(System.currentTimeMillis() - startTime);
+  }
+
   private boolean skipCurrentTTLAndModificationCheck() {
     if (this.databaseName.equals(InformationSchema.INFORMATION_DATABASE)) {
       return true;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java

@@ -27,7 +27,6 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICrossCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
@@ -193,11 +192,6 @@ public boolean doCompaction() {
         performer.setTargetFiles(targetTsfileResourceList);
         performer.setSummary(summary);
         performer.perform();
-        if (performer instanceof ReadPointCompactionPerformer) {
-          for (TsFileResource resource : getAllSourceTsFiles()) {
-            CompactionUtils.removeDeletedObjectFiles(resource);
-          }
-        }
 
         CompactionUtils.updateProgressIndexAndMark(
             targetTsfileResourceList, selectedSequenceFiles, selectedUnsequenceFiles);

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java

@@ -30,7 +30,6 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.IInnerCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
@@ -390,11 +389,6 @@ protected void compact(SimpleCompactionLogger compactionLogger) throws Exception
     performer.setTargetFiles(filesView.targetFilesInPerformer);
     performer.setSummary(summary);
     performer.perform();
-    if (performer instanceof ReadPointCompactionPerformer) {
-      for (TsFileResource resource : filesView.sourceFilesInCompactionPerformer) {
-        CompactionUtils.removeDeletedObjectFiles(resource);
-      }
-    }
 
     prepareTargetFiles();
 

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java

@@ -219,7 +219,6 @@ public boolean settleWithFullyDirtyFiles() throws IllegalPathException, IOExcept
       if (recoverMemoryStatus) {
         tsFileManager.remove(resource, resource.isSeq());
       }
-      CompactionUtils.removeDeletedObjectFiles(resource);
 
       boolean res = deleteTsFileOnDisk(resource);
       if (res) {