Some files were not added to TsFileResourceManager

Author: shuwenwei

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

@@ -3063,9 +3063,6 @@ private boolean loadTsFileToUnSequence(
     PipeInsertionDataNodeListener.getInstance()
         .listenToTsFile(dataRegionId, databaseName, tsFileResource, true, isGeneratedByPipe);
 
-    // help tsfile resource degrade
-    tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
-
     tsFileManager.add(tsFileResource, false);
 
     return true;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java

@@ -174,7 +174,6 @@ public void removeAll(List<TsFileResource> tsFileResourceList, boolean sequence)
     try {
       for (TsFileResource resource : tsFileResourceList) {
         remove(resource, sequence);
-        TsFileResourceManager.getInstance().removeTsFileResource(resource);
       }
     } finally {
       writeLock("removeAll");
@@ -187,6 +186,7 @@ public void removeAll(List<TsFileResource> tsFileResourceList, boolean sequence)
    */
   public void insertToPartitionFileList(
       TsFileResource tsFileResource, long timePartition, boolean sequence, int insertPos) {
+    registerTsFileResourceToResourceManager(tsFileResource);
     writeLock("add");
     try {
       Map<Long, TsFileResourceList> selectedMap = sequence ? sequenceFiles : unsequenceFiles;
@@ -204,6 +204,7 @@ public void insertToPartitionFileList(
   }
 
   public void add(TsFileResource tsFileResource, boolean sequence) {
+    registerTsFileResourceToResourceManager(tsFileResource);
     writeLock("add");
     try {
       Map<Long, TsFileResourceList> selectedMap = sequence ? sequenceFiles : unsequenceFiles;
@@ -221,6 +222,7 @@ public void add(TsFileResource tsFileResource, boolean sequence) {
   }
 
   public void keepOrderInsert(TsFileResource tsFileResource, boolean sequence) throws IOException {
+    registerTsFileResourceToResourceManager(tsFileResource);
     writeLock("keepOrderInsert");
     try {
       Map<Long, TsFileResourceList> selectedMap = sequence ? sequenceFiles : unsequenceFiles;
@@ -248,6 +250,13 @@ public void addAll(List<TsFileResource> tsFileResourceList, boolean sequence) {
     }
   }
 
+  private void registerTsFileResourceToResourceManager(TsFileResource resource) {
+    TsFileResourceStatus status = resource.getStatus();
+    if (status != TsFileResourceStatus.UNCLOSED && status != TsFileResourceStatus.DELETED) {
+      TsFileResourceManager.getInstance().registerSealedTsFileResource(resource);
+    }
+  }
+
   /** This method is called after compaction to update memory. */
   public void replace(
       List<TsFileResource> seqFileResources,

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java

@@ -72,6 +72,7 @@ public void setUp()
     enableInsertionCrossSpaceCompaction =
         IoTDBDescriptor.getInstance().getConfig().isEnableCrossSpaceCompaction();
     IoTDBDescriptor.getInstance().getConfig().setEnableCrossSpaceCompaction(true);
+    TsFileResourceManager.getInstance().clear();
   }
 
   @After
@@ -80,6 +81,7 @@ public void tearDown() throws IOException, StorageEngineException {
         .getConfig()
         .setEnableCrossSpaceCompaction(enableInsertionCrossSpaceCompaction);
     super.tearDown();
+    TsFileResourceManager.getInstance().clear();
   }
 
   @Test
@@ -127,6 +129,9 @@ public void test1() throws IOException, InterruptedException {
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
     Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty());
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -168,6 +173,9 @@ public void test2() throws IOException, InterruptedException {
     TsFileResource targetFile = tsFileManager.getTsFileList(true).get(1);
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -209,6 +217,9 @@ public void test3() throws IOException, InterruptedException {
     TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0);
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(0, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -256,6 +267,9 @@ public void test4() throws IOException, InterruptedException {
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(4, timestamp);
     Assert.assertTrue(tsFileManager.getTsFileList(false).isEmpty());
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -291,6 +305,9 @@ public void test5() throws IOException, InterruptedException {
     TsFileResource targetFile = tsFileManager.getTsFileList(true).get(0);
     long timestamp = TsFileNameGenerator.getTsFileName(targetFile.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -340,6 +357,9 @@ public void testInsertionCompactionSchedule() throws IOException, InterruptedExc
     TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(2);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime();
     Assert.assertEquals(6, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -378,6 +398,9 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace()
     TsFileResource targetFile3 = tsFileManager.getTsFileList(true).get(2);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile3.getTsFile().getName()).getTime();
     Assert.assertEquals(4, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -423,6 +446,9 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace2()
     TsFileResource targetFile2 = tsFileManager.getTsFileList(true).get(1);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile2.getTsFile().getName()).getTime();
     Assert.assertEquals(2, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -473,6 +499,9 @@ public void testInsertionCompactionScheduleWithMultiTimePartitions()
     TsFileResource targetFile3 = tsFileManager.getOrCreateSequenceListByTimePartition(2808).get(0);
     timestamp = TsFileNameGenerator.getTsFileName(targetFile3.getTsFile().getName()).getTime();
     Assert.assertEquals(4, timestamp);
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   @Test
@@ -526,6 +555,9 @@ public void testInsertionCompactionUpdateFileMetrics() throws IOException {
     Assert.assertEquals(seqFileNumBeforeCompaction, FileMetrics.getInstance().getFileCount(true));
     Assert.assertEquals(
         unseqFileNumBeforeCompaction, FileMetrics.getInstance().getFileCount(false));
+    Assert.assertEquals(
+        tsFileManager.size(true) + tsFileManager.size(false),
+        TsFileResourceManager.getInstance().getPriorityQueueSize());
   }
 
   public TsFileResource generateSingleNonAlignedSeriesFileWithDevices(