Skip to content

Commit b3509d6

Browse files
authored
Load: Fixed parameter passing error in MoveFile function (apache#16748)
* Load: Fixed parameter passing error in MoveFile function * update * add ut * update
1 parent 0f2483f commit b3509d6

File tree

2 files changed

+132
-39
lines changed

2 files changed

+132
-39
lines changed

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java

Lines changed: 53 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ public static void moveFileWithMD5Check(final File sourceFile, final File target
418418
final String sourceFileName = sourceFile.getName();
419419
final File targetFile = new File(targetDir, sourceFileName);
420420
if (targetFile.exists()) {
421-
moveFile(sourceFile, targetFile);
421+
moveFile(sourceFile, targetDir);
422422
} else {
423423
org.apache.tsfile.external.commons.io.FileUtils.moveFileToDirectory(
424424
sourceFile, targetDir, true);
@@ -428,31 +428,37 @@ public static void moveFileWithMD5Check(final File sourceFile, final File target
428428
private static void moveFile(File sourceFile, File targetDir) throws IOException {
429429
String sourceFileName = sourceFile.getName();
430430
final File exitsFile = new File(targetDir, sourceFileName);
431-
try (final FileInputStream is1 = new FileInputStream(sourceFile);
432-
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
433-
long sourceFileSize = is1.getChannel().size();
434-
long exitsFileSize = is2.getChannel().size();
435-
if (sourceFileSize != exitsFileSize) {
436-
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
437-
if (!file.exists()) {
438-
moveFileRename(sourceFile, file);
439-
return;
440-
}
441-
}
442431

443-
String sourceFileMD5 = DigestUtils.md5Hex(is1);
444-
String exitsFileMD5 = DigestUtils.md5Hex(is2);
432+
// First check file sizes
433+
long sourceFileSize = sourceFile.length();
434+
long existsFileSize = exitsFile.length();
445435

446-
if (sourceFileMD5.equals(exitsFileMD5)) {
447-
org.apache.tsfile.external.commons.io.FileUtils.forceDelete(sourceFile);
448-
LOGGER.info(
449-
"Deleted the file {} because it already exists in the target directory: {}",
450-
sourceFile.getName(),
451-
targetDir.getAbsolutePath());
452-
} else {
453-
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
436+
if (sourceFileSize != existsFileSize) {
437+
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
438+
if (!file.exists()) {
454439
moveFileRename(sourceFile, file);
455440
}
441+
return;
442+
}
443+
444+
// If sizes are equal, check MD5
445+
String sourceFileMD5;
446+
String existsFileMD5;
447+
try (final FileInputStream is1 = new FileInputStream(sourceFile);
448+
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
449+
sourceFileMD5 = DigestUtils.md5Hex(is1);
450+
existsFileMD5 = DigestUtils.md5Hex(is2);
451+
}
452+
453+
if (sourceFileMD5.equals(existsFileMD5)) {
454+
org.apache.tsfile.external.commons.io.FileUtils.forceDelete(sourceFile);
455+
LOGGER.info(
456+
"Deleted the file {} because it already exists in the target directory: {}",
457+
sourceFile.getName(),
458+
targetDir.getAbsolutePath());
459+
} else {
460+
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
461+
moveFileRename(sourceFile, file);
456462
}
457463
}
458464

@@ -492,27 +498,35 @@ private static void copyFileWithMD5(final File sourceFile, final File targetDir)
492498
throws IOException {
493499
String sourceFileName = sourceFile.getName();
494500
final File exitsFile = new File(targetDir, sourceFileName);
495-
try (final FileInputStream is1 = new FileInputStream(sourceFile);
496-
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
497-
long sourceFileSize = is1.getChannel().size();
498-
long exitsFileSize = is2.getChannel().size();
499-
if (sourceFileSize != exitsFileSize) {
500-
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
501-
if (!file.exists()) {
502-
copyFileRename(sourceFile, file);
503-
return;
504-
}
505-
}
506-
String sourceFileMD5 = DigestUtils.md5Hex(is1);
507-
String exitsFileMD5 = DigestUtils.md5Hex(is2);
508-
if (sourceFileMD5.equals(exitsFileMD5)) {
509-
return;
510-
}
511501

512-
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
502+
// First check file sizes
503+
long sourceFileSize = sourceFile.length();
504+
long exitsFileSize = exitsFile.length();
505+
506+
if (sourceFileSize != exitsFileSize) {
507+
File file = renameWithSize(sourceFile, sourceFileSize, targetDir);
513508
if (!file.exists()) {
514509
copyFileRename(sourceFile, file);
515510
}
511+
return;
512+
}
513+
514+
// If sizes are equal, check MD5
515+
String sourceFileMD5;
516+
String exitsFileMD5;
517+
try (final FileInputStream is1 = new FileInputStream(sourceFile);
518+
final FileInputStream is2 = new FileInputStream(exitsFile); ) {
519+
sourceFileMD5 = DigestUtils.md5Hex(is1);
520+
exitsFileMD5 = DigestUtils.md5Hex(is2);
521+
}
522+
523+
if (sourceFileMD5.equals(exitsFileMD5)) {
524+
return;
525+
}
526+
527+
File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir);
528+
if (!file.exists()) {
529+
copyFileRename(sourceFile, file);
516530
}
517531
}
518532

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.commons.utils;
21+
22+
import org.apache.tsfile.enums.TSDataType;
23+
import org.apache.tsfile.exception.write.WriteProcessException;
24+
import org.apache.tsfile.write.TsFileWriter;
25+
import org.apache.tsfile.write.record.Tablet;
26+
import org.apache.tsfile.write.schema.MeasurementSchema;
27+
import org.junit.After;
28+
import org.junit.Before;
29+
import org.junit.Test;
30+
31+
import java.io.File;
32+
import java.io.IOException;
33+
import java.nio.file.Files;
34+
import java.util.Collections;
35+
36+
public class FileUtilsTest {
37+
private File tmpDir;
38+
private File targetDir;
39+
40+
@Before
41+
public void setUp() throws Exception {
42+
tmpDir = new File(Files.createTempDirectory("load").toUri());
43+
targetDir = new File(Files.createTempDirectory("target").toUri());
44+
}
45+
46+
@After
47+
public void tearDown() throws Exception {
48+
tmpDir.delete();
49+
targetDir.delete();
50+
}
51+
52+
@Test
53+
public void testFileUtils() throws WriteProcessException, IOException {
54+
File tstFile = new File(tmpDir, "1-1-0-0.tsfile");
55+
File tstFile2 = new File(tmpDir, "2-1-0-0.tsfile");
56+
generateFile(tstFile);
57+
FileUtils.copyFile(tstFile, tstFile2);
58+
FileUtils.moveFileWithMD5Check(tstFile, targetDir);
59+
tstFile2.renameTo(tstFile);
60+
FileUtils.moveFileWithMD5Check(tstFile, targetDir);
61+
}
62+
63+
private void generateFile(File tsfile) throws WriteProcessException, IOException {
64+
try (TsFileWriter writer = new TsFileWriter(tsfile)) {
65+
writer.registerAlignedTimeseries(
66+
"root.test.d1",
67+
Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN)));
68+
Tablet tablet =
69+
new Tablet(
70+
"root.test.d1",
71+
Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN)));
72+
for (int i = 0; i < 5; i++) {
73+
tablet.addTimestamp(i, i);
74+
tablet.addValue(i, 0, true);
75+
}
76+
writer.writeTree(tablet);
77+
}
78+
}
79+
}

0 commit comments

Comments
 (0)