Skip to content

Commit 1db0bc8

Browse files
authored
[To dev/1.3] Load : Fixed parameter passing error in MoveFile function (#16748) (#16792)
* Load: Fixed the issue of TSFile parent directory being null and TSFile resource being updated during the Load process. (#16751) * fix * fix
1 parent 65ebd7c commit 1db0bc8

File tree

2 files changed

+133
-39
lines changed

2 files changed

+133
-39
lines changed

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java

Lines changed: 53 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -400,32 +400,38 @@ public static void moveFileWithMD5Check(final File sourceFile, final File target
400400

401401
private static void moveFile(File sourceFile, File targetDir) throws IOException {
402402
String sourceFileName = sourceFile.getName();
403-
final File existsFile = new File(targetDir, sourceFileName);
404-
try (final FileInputStream is1 = new FileInputStream(sourceFile);
405-
final FileInputStream is2 = new FileInputStream(existsFile); ) {
406-
long sourceFileSize = is1.getChannel().size();
407-
long exitsFileSize = is2.getChannel().size();
408-
if (sourceFileSize != exitsFileSize) {
409-
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
410-
if (!file.exists()) {
411-
moveFileRename(sourceFile, file);
412-
return;
413-
}
414-
}
403+
final File exitsFile = new File(targetDir, sourceFileName);
415404

416-
String sourceFileMD5 = DigestUtils.md5Hex(is1);
417-
String existsFileMD5 = DigestUtils.md5Hex(is2);
405+
// First check file sizes
406+
long sourceFileSize = sourceFile.length();
407+
long existsFileSize = exitsFile.length();
418408

419-
if (sourceFileMD5.equals(existsFileMD5)) {
420-
org.apache.commons.io.FileUtils.forceDelete(sourceFile);
421-
LOGGER.info(
422-
"Deleted the file {} because it already exists in the target directory: {}",
423-
sourceFile.getName(),
424-
targetDir.getAbsolutePath());
425-
} else {
426-
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
409+
if (sourceFileSize != existsFileSize) {
410+
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
411+
if (!file.exists()) {
427412
moveFileRename(sourceFile, file);
428413
}
414+
return;
415+
}
416+
417+
// If sizes are equal, check MD5
418+
String sourceFileMD5;
419+
String existsFileMD5;
420+
try (final FileInputStream is1 = new FileInputStream(sourceFile);
421+
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
422+
sourceFileMD5 = DigestUtils.md5Hex(is1);
423+
existsFileMD5 = DigestUtils.md5Hex(is2);
424+
}
425+
426+
if (sourceFileMD5.equals(existsFileMD5)) {
427+
org.apache.commons.io.FileUtils.forceDelete(sourceFile);
428+
LOGGER.info(
429+
"Deleted the file {} because it already exists in the target directory: {}",
430+
sourceFile.getName(),
431+
targetDir.getAbsolutePath());
432+
} else {
433+
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
434+
moveFileRename(sourceFile, file);
429435
}
430436
}
431437

@@ -465,27 +471,35 @@ private static void copyFileWithMD5(final File sourceFile, final File targetDir)
465471
throws IOException {
466472
String sourceFileName = sourceFile.getName();
467473
final File exitsFile = new File(targetDir, sourceFileName);
468-
try (final FileInputStream is1 = new FileInputStream(sourceFile);
469-
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
470-
long sourceFileSize = is1.getChannel().size();
471-
long exitsFileSize = is2.getChannel().size();
472-
if (sourceFileSize != exitsFileSize) {
473-
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
474-
if (!file.exists()) {
475-
copyFileRename(sourceFile, file);
476-
return;
477-
}
478-
}
479-
String sourceFileMD5 = DigestUtils.md5Hex(is1);
480-
String exitsFileMD5 = DigestUtils.md5Hex(is2);
481-
if (sourceFileMD5.equals(exitsFileMD5)) {
482-
return;
483-
}
484474

485-
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
475+
// First check file sizes
476+
long sourceFileSize = sourceFile.length();
477+
long exitsFileSize = exitsFile.length();
478+
479+
if (sourceFileSize != exitsFileSize) {
480+
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
486481
if (!file.exists()) {
487482
copyFileRename(sourceFile, file);
488483
}
484+
return;
485+
}
486+
487+
// If sizes are equal, check MD5
488+
String sourceFileMD5;
489+
String exitsFileMD5;
490+
try (final FileInputStream is1 = new FileInputStream(sourceFile);
491+
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
492+
sourceFileMD5 = DigestUtils.md5Hex(is1);
493+
exitsFileMD5 = DigestUtils.md5Hex(is2);
494+
}
495+
496+
if (sourceFileMD5.equals(exitsFileMD5)) {
497+
return;
498+
}
499+
500+
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
501+
if (!file.exists()) {
502+
copyFileRename(sourceFile, file);
489503
}
490504
}
491505

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.commons.utils;
21+
22+
import org.apache.tsfile.enums.TSDataType;
23+
import org.apache.tsfile.exception.write.WriteProcessException;
24+
import org.apache.tsfile.read.common.Path;
25+
import org.apache.tsfile.write.TsFileWriter;
26+
import org.apache.tsfile.write.record.Tablet;
27+
import org.apache.tsfile.write.schema.MeasurementSchema;
28+
import org.junit.After;
29+
import org.junit.Before;
30+
import org.junit.Test;
31+
32+
import java.io.File;
33+
import java.io.IOException;
34+
import java.nio.file.Files;
35+
import java.util.Collections;
36+
37+
public class FileUtilsTest {
38+
private File tmpDir;
39+
private File targetDir;
40+
41+
@Before
42+
public void setUp() throws Exception {
43+
tmpDir = new File(Files.createTempDirectory("load").toUri());
44+
targetDir = new File(Files.createTempDirectory("target").toUri());
45+
}
46+
47+
@After
48+
public void tearDown() throws Exception {
49+
tmpDir.delete();
50+
targetDir.delete();
51+
}
52+
53+
@Test
54+
public void testFileUtils() throws WriteProcessException, IOException {
55+
File tstFile = new File(tmpDir, "1-1-0-0.tsfile");
56+
File tstFile2 = new File(tmpDir, "2-1-0-0.tsfile");
57+
generateFile(tstFile);
58+
FileUtils.copyFile(tstFile, tstFile2);
59+
FileUtils.moveFileWithMD5Check(tstFile, targetDir);
60+
tstFile2.renameTo(tstFile);
61+
FileUtils.moveFileWithMD5Check(tstFile, targetDir);
62+
}
63+
64+
private void generateFile(File tsfile) throws WriteProcessException, IOException {
65+
try (TsFileWriter writer = new TsFileWriter(tsfile)) {
66+
writer.registerAlignedTimeseries(
67+
new Path("root.test.d1"),
68+
Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN)));
69+
Tablet tablet =
70+
new Tablet(
71+
"root.test.d1",
72+
Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN)));
73+
for (int i = 0; i < 5; i++) {
74+
tablet.addTimestamp(i, i);
75+
tablet.addValue("s1", 0, true);
76+
}
77+
writer.writeAligned(tablet);
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)