Skip to content

Commit e2383e1

Browse files
HTHouHTHou
committed
dev
1 parent ee39ee1 commit e2383e1

File tree

3 files changed

+87
-9
lines changed

3 files changed

+87
-9
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 60 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -158,18 +158,21 @@
158158
import org.apache.iotdb.db.utils.DateTimeUtils;
159159
import org.apache.iotdb.db.utils.EncryptDBUtils;
160160
import org.apache.iotdb.db.utils.ModificationUtils;
161+
import org.apache.iotdb.db.utils.ObjectTypeUtils;
161162
import org.apache.iotdb.db.utils.ObjectWriter;
162163
import org.apache.iotdb.metrics.utils.MetricLevel;
163164
import org.apache.iotdb.rpc.RpcUtils;
164165
import org.apache.iotdb.rpc.TSStatusCode;
165166

166167
import com.github.benmanes.caffeine.cache.Cache;
167168
import com.github.benmanes.caffeine.cache.Caffeine;
169+
import com.google.common.io.BaseEncoding;
168170
import org.apache.thrift.TException;
169171
import org.apache.tsfile.external.commons.io.FileUtils;
170172
import org.apache.tsfile.external.commons.lang3.tuple.Triple;
171173
import org.apache.tsfile.file.metadata.ChunkMetadata;
172174
import org.apache.tsfile.file.metadata.IDeviceID;
175+
import org.apache.tsfile.file.metadata.IDeviceID.Factory;
173176
import org.apache.tsfile.file.metadata.TableSchema;
174177
import org.apache.tsfile.fileSystem.FSFactoryProducer;
175178
import org.apache.tsfile.fileSystem.FSType;
@@ -186,6 +189,7 @@
186189
import java.io.File;
187190
import java.io.IOException;
188191
import java.nio.ByteBuffer;
192+
import java.nio.charset.StandardCharsets;
189193
import java.nio.file.Files;
190194
import java.nio.file.Path;
191195
import java.nio.file.Paths;
@@ -2762,8 +2766,10 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException {
27622766
}
27632767
}
27642768

2765-
if (TierManager.getInstance().checkObjectPathExist(dataRegionIdString, tableName)) {
2766-
deleteObjectFiles(tableName, modEntries);
2769+
List<File> matchedObjectDirs =
2770+
TierManager.getInstance().getAllMatchedObjectDirs(dataRegionIdString, tableName);
2771+
if (!matchedObjectDirs.isEmpty()) {
2772+
deleteObjectFiles(matchedObjectDirs, modEntries);
27672773
}
27682774

27692775
List<List<TsFileResource>> sealedTsFileResourceLists = new ArrayList<>(modEntries.size());
@@ -2934,9 +2940,58 @@ private List<WALFlushListener> logDeletionInWAL(
29342940
return walFlushListeners;
29352941
}
29362942

2937-
private void deleteObjectFiles(String tableName, List<TableDeletionEntry> modEntries) {
2938-
for (TableDeletionEntry modEntry : modEntries) {
2939-
2943+
private void deleteObjectFiles(List<File> matchedObjectDirs, List<TableDeletionEntry> modEntries)
2944+
throws IOException {
2945+
for (File matchedObjectDir : matchedObjectDirs) {
2946+
try (Stream<Path> paths = Files.walk(matchedObjectDir.toPath())) {
2947+
paths
2948+
.filter(Files::isRegularFile)
2949+
.filter(
2950+
path -> {
2951+
String name = path.getFileName().toString();
2952+
return name.endsWith(".bin");
2953+
})
2954+
.forEach(
2955+
path -> {
2956+
Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path);
2957+
String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2];
2958+
for (int i = 0; i < ideviceIdSegments.length; i++) {
2959+
ideviceIdSegments[i] =
2960+
config.getRestrictObjectLimit()
2961+
? relativePath.getName(i).toString()
2962+
: new String(
2963+
BaseEncoding.base32()
2964+
.omitPadding()
2965+
.decode(relativePath.getName(i).toString()),
2966+
StandardCharsets.UTF_8);
2967+
}
2968+
IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments);
2969+
String measurementId =
2970+
config.getRestrictObjectLimit()
2971+
? relativePath.getName(relativePath.getNameCount() - 2).toString()
2972+
: new String(
2973+
BaseEncoding.base32()
2974+
.omitPadding()
2975+
.decode(
2976+
relativePath
2977+
.getName(relativePath.getNameCount() - 2)
2978+
.toString()),
2979+
StandardCharsets.UTF_8);
2980+
String fileName = path.getFileName().toString();
2981+
long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.')));
2982+
logger.info(
2983+
"timestamp {}, measurementId {}, ideviceId {}",
2984+
timestamp,
2985+
measurementId,
2986+
iDeviceID);
2987+
for (TableDeletionEntry modEntry : modEntries) {
2988+
if (modEntry.affects(iDeviceID, timestamp, timestamp)
2989+
&& modEntry.affects(measurementId)) {
2990+
ObjectTypeUtils.deleteObjectPath(path.toFile());
2991+
}
2992+
}
2993+
});
2994+
}
29402995
}
29412996
}
29422997

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ public Optional<File> getAbsoluteObjectFilePath(String filePath, boolean needTem
274274
return Optional.empty();
275275
}
276276

277-
public boolean checkObjectPathExist(String regionIdStr, String... path) {
277+
public List<File> getAllMatchedObjectDirs(String regionIdStr, String... path) {
278+
List<File> matchedDirs = new ArrayList<>();
278279
StringBuilder objectPath = new StringBuilder();
279280
objectPath.append(regionIdStr);
280281
for (String str : path) {
@@ -288,12 +289,13 @@ public boolean checkObjectPathExist(String regionIdStr, String... path) {
288289
.encode(str.getBytes(StandardCharsets.UTF_8)));
289290
}
290291
for (String objectDir : objectDirs) {
291-
File objectFilePath = FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString());
292+
File objectFilePath =
293+
FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString());
292294
if (objectFilePath.exists()) {
293-
return true;
295+
matchedDirs.add(objectFilePath);
294296
}
295297
}
296-
return false;
298+
return matchedDirs;
297299
}
298300

299301
public int getTiersNum() {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,27 @@ public static void deleteObjectPathFromBinary(Binary binary) {
294294
}
295295
}
296296

297+
public static void deleteObjectPath(File file) {
298+
File tmpFile = new File(file.getPath() + ".tmp");
299+
File bakFile = new File(file.getPath() + ".back");
300+
for (int i = 0; i < 2; i++) {
301+
if (file.exists()) {
302+
FileMetrics.getInstance().decreaseObjectFileNum(1);
303+
FileMetrics.getInstance().decreaseObjectFileSize(file.length());
304+
}
305+
try {
306+
deleteObjectFile(file);
307+
deleteObjectFile(tmpFile);
308+
deleteObjectFile(bakFile);
309+
} catch (IOException e) {
310+
logger.error("Failed to remove object file {}", file.getAbsolutePath(), e);
311+
}
312+
}
313+
if (file.getParentFile().exists()) {
314+
315+
}
316+
}
317+
297318
private static void deleteObjectFile(File file) throws IOException {
298319
if (file.exists()) {
299320
logger.info("Remove object file {}, size is {}(byte)", file.getAbsolutePath(), file.length());

0 commit comments

Comments
 (0)