Skip to content

Commit d71abcf

Browse files
author
beanwang
committed
support upload file by cosn fs
1 parent 9dba35c commit d71abcf

File tree

12 files changed

+688
-27
lines changed

12 files changed

+688
-27
lines changed

‎dep/cos_migrate_tool-1.4.13-jar-with-dependencies.jar‎ renamed to ‎dep/cos_migrate_tool-1.4.13.1-jar-with-dependencies.jar‎

30.8 MB
Binary file not shown.

‎pom.xml‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.qcloud</groupId>
55
<artifactId>cos_migrate_tool</artifactId>
6-
<version>1.4.13</version>
6+
<version>1.4.13.1</version>
77
<packaging>jar</packaging>
88
<properties>
99
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -96,6 +96,31 @@
9696
</exclusion>
9797
</exclusions>
9898
</dependency>
99+
100+
<dependency>
101+
<groupId>com.qcloud.cos</groupId>
102+
<artifactId>hadoop-cos</artifactId>
103+
<version>8.2.7</version>
104+
</dependency>
105+
106+
<dependency>
107+
<groupId>com.qcloud</groupId>
108+
<artifactId>chdfs_hadoop_plugin_network</artifactId>
109+
<version>2.8</version>
110+
</dependency>
111+
112+
<dependency>
113+
<groupId>com.qcloud</groupId>
114+
<artifactId>cos_api-bundle</artifactId>
115+
<version>5.6.112</version>
116+
</dependency>
117+
118+
<dependency>
119+
<groupId>org.apache.hadoop</groupId>
120+
<artifactId>hadoop-common</artifactId>
121+
<version>2.8.5</version>
122+
</dependency>
123+
99124
</dependencies>
100125

101126
<build>

‎src/main/java/com/qcloud/cos_migrate_tool/app/App.java‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,7 @@
77
import java.io.InputStreamReader;
88

99
import com.qcloud.cos.internal.SkipMd5CheckStrategy;
10-
import com.qcloud.cos_migrate_tool.config.CommonConfig;
11-
import com.qcloud.cos_migrate_tool.config.ConfigParser;
12-
import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
13-
import com.qcloud.cos_migrate_tool.config.CopyFromAliConfig;
14-
import com.qcloud.cos_migrate_tool.config.CopyFromAwsConfig;
15-
import com.qcloud.cos_migrate_tool.config.CopyFromCompetitorConfig;
16-
import com.qcloud.cos_migrate_tool.config.CopyFromCspConfig;
17-
import com.qcloud.cos_migrate_tool.config.CopyFromLocalConfig;
18-
import com.qcloud.cos_migrate_tool.config.CopyFromQiniuConfig;
19-
import com.qcloud.cos_migrate_tool.config.CopyFromUpyunConfig;
20-
import com.qcloud.cos_migrate_tool.config.CopyFromUrllistConfig;
21-
import com.qcloud.cos_migrate_tool.config.MigrateType;
10+
import com.qcloud.cos_migrate_tool.config.*;
2211
import com.qcloud.cos_migrate_tool.meta.TaskStatics;
2312
import com.qcloud.cos_migrate_tool.task.MigrateAliTaskExecutor;
2413
import com.qcloud.cos_migrate_tool.task.MigrateAwsTaskExecutor;
@@ -31,6 +20,7 @@
3120
import com.qcloud.cos_migrate_tool.task.MigrateUrllistTaskExecutor;
3221
import com.qcloud.cos_migrate_tool.task.TaskExecutor;
3322

23+
import com.qcloud.cos_migrate_tool.task_by_hadoop_fs.MigrateLocalToCosnTaskExecutor;
3424
import org.slf4j.Logger;
3525
import org.slf4j.LoggerFactory;
3626

@@ -57,6 +47,8 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
5747
return new MigrateCspTaskExecutor((CopyFromCspConfig) config);
5848
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_UPYUN)) {
5949
return new MigrateUpyunTaskExecutor((CopyFromUpyunConfig) config);
50+
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)) {
51+
return new MigrateLocalToCosnTaskExecutor((CopyFromLocalToCosnConfig) config);
6052
} else {
6153
System.out.println("unknown migrate type");
6254
}

‎src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ public class ConfigParser {
7676
private static final String FILE_LIST_PATH = "fileListPath";
7777
private static final String CHECK_LOCAL_RECORD = "checkLocalRecord";
7878

79+
private static final String LOCAL_TO_COSN_FS_SECTION_NAME = "migrateLocalToCosnFs";
80+
7981
private static final String ALI_SECTION_NAME = "migrateAli";
8082
private static final String AWS_SECTION_NAME = "migrateAws";
8183
private static final String QINIU_SECTION_NAME = "migrateQiniu";
@@ -303,6 +305,14 @@ public boolean parse() {
303305
if (!initCopyFromUpyunConfig(prefs, (CopyFromUpyunConfig) config)) {
304306
return false;
305307
}
308+
} else if (migrateType.equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)){
309+
if (!checkMigrateLocalToCosnFsConfig(prefs)) {
310+
return false;
311+
}
312+
config = new CopyFromLocalToCosnConfig();
313+
if (!initCopyFromLocalToCosnConfig(prefs, (CopyFromLocalToCosnConfig) config)) {
314+
return false;
315+
}
306316
}
307317

308318

@@ -402,6 +412,13 @@ private boolean checkMigrateLocalConfig(Preferences prefs) {
402412
return true;
403413
}
404414

415+
private boolean checkMigrateLocalToCosnFsConfig(Preferences prefs) {
416+
if (!isKeyExist(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH)) {
417+
return false;
418+
}
419+
return true;
420+
}
421+
405422
private boolean checkMigrateCopyBucketConfig(Preferences prefs) {
406423
if (!isKeyExist(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_REGION)) {
407424
return false;
@@ -715,6 +732,69 @@ private boolean initCopyFromLocalConfig(Preferences prefs,
715732
return true;
716733
}
717734

735+
private boolean initCopyFromLocalToCosnConfig(Preferences prefs,
736+
CopyFromLocalToCosnConfig copyLocalConfig) {
737+
if (!initCommonConfig(prefs, copyLocalConfig)) {
738+
return false;
739+
}
740+
try {
741+
742+
String localPathConfig = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH);
743+
assert (localPathConfig != null);
744+
copyLocalConfig.setLocalPath(localPathConfig);
745+
746+
String excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_EXECLUDE);
747+
if (excludes != null && !excludes.trim().isEmpty()) {
748+
copyLocalConfig.setExcludes(excludes);
749+
} else {
750+
excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, "excludes");
751+
if (excludes != null && !excludes.trim().isEmpty()) {
752+
copyLocalConfig.setExcludes(excludes);
753+
}
754+
}
755+
756+
String ignoreModifiedTimeLessThanStr =
757+
getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_MODIFIED_TIME_LESS_THAN);
758+
if (ignoreModifiedTimeLessThanStr != null
759+
&& !ignoreModifiedTimeLessThanStr.trim().isEmpty()) {
760+
copyLocalConfig.setIgnoreModifiedTimeLessThan(ignoreModifiedTimeLessThanStr);
761+
}
762+
763+
String ignoreSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_SUFFIX);
764+
if (ignoreSuffix != null && !ignoreSuffix.trim().isEmpty()) {
765+
copyLocalConfig.setIgnoreSuffix(ignoreSuffix);
766+
}
767+
768+
String includeSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, INCLUDE_SUFFIX);
769+
if (includeSuffix != null && !includeSuffix.trim().isEmpty()) {
770+
copyLocalConfig.setIncludeSuffix(includeSuffix);
771+
}
772+
773+
String ignoreEmptyFile = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_EMPTY_FILE);
774+
if (ignoreEmptyFile != null && (ignoreEmptyFile.compareToIgnoreCase("on") == 0)) {
775+
copyLocalConfig.setIgnoreEmptyFile(true);
776+
}
777+
String fileListMode = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_MODE);
778+
if (fileListMode != null && (fileListMode.compareToIgnoreCase("on") == 0)) {
779+
copyLocalConfig.setFileListMode(true);
780+
}
781+
String fileListPath = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_PATH);
782+
if (fileListPath != null) {
783+
copyLocalConfig.setFileListPath(fileListPath);
784+
}
785+
786+
String strCheckLocal = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, CHECK_LOCAL_RECORD);
787+
if (strCheckLocal != null) {
788+
copyLocalConfig.setCheckLocalRecord(strCheckLocal);
789+
}
790+
} catch (Exception e) {
791+
System.err.println(e.getMessage());
792+
log.error(e.getMessage());
793+
return false;
794+
}
795+
return true;
796+
}
797+
718798
private boolean initCopyFromUrllistConfig(Preferences prefs,
719799
CopyFromUrllistConfig copyUrllistConfig) {
720800
if (!initCommonConfig(prefs, copyUrllistConfig)) {

‎src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalConfig.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ public void setIgnoreModifiedTimeLessThan(String ignoreModifiedTimeLessThanStr)
112112
public long getIgnoreModifiedTimeLessThan() {
113113
return ignoreModifiedTimeLessThan;
114114
}
115+
115116
public boolean isFileListMode() {
116117
return fileListMode;
117118
}

‎src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalToCosnConfig.java‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package com.qcloud.cos_migrate_tool.config;
2+
3+
import com.qcloud.cos_migrate_tool.utils.SystemUtils;
4+
5+
import java.io.File;
6+
import java.nio.file.Path;
7+
import java.util.HashSet;
8+
import java.util.Set;
9+
10+
public class CopyFromLocalToCosnConfig extends CommonConfig{
11+
private String localPath;
12+
private Set<String> excludes = new HashSet<String>();
13+
private long ignoreModifiedTimeLessThan = -1;
14+
private Set<String> ignoreSuffixs = new HashSet<String>();
15+
private Set<String> includeSuffixs = new HashSet<String>();
16+
boolean ignoreEmptyFile = false;
17+
boolean fileListMode = false;
18+
private String fileListPath;
19+
private boolean isCheckLocalRecord = true;
20+
21+
public void setIgnoreEmptyFile(boolean ignoreEmptyFile) {
22+
this.ignoreEmptyFile = ignoreEmptyFile;
23+
}
24+
25+
public void setIgnoreSuffix(String ignore) {
26+
ignore = ignore.trim();
27+
String[] ignoreArray = ignore.split(";");
28+
for (String ignoreElement : ignoreArray) {
29+
this.ignoreSuffixs.add(ignoreElement);
30+
}
31+
}
32+
33+
public void setIncludeSuffix(String include) {
34+
include = include.trim();
35+
String[] includeArray = include.split(";");
36+
for (String includeElement : includeArray) {
37+
this.includeSuffixs.add(includeElement);
38+
}
39+
}
40+
41+
public String needToMigrate(Path file, String localPath) {
42+
if (isExcludes(localPath)) {
43+
return "excludes";
44+
}
45+
46+
for (String suffix:ignoreSuffixs) {
47+
if(localPath.endsWith(suffix)) {
48+
return "suffix";
49+
}
50+
}
51+
52+
if (ignoreEmptyFile) {
53+
File localFile = new File(file.toString());
54+
if (localFile.length() == 0) {
55+
return "empty file";
56+
}
57+
}
58+
59+
if (includeSuffixs.isEmpty()) {
60+
return "";
61+
}
62+
63+
for (String suffix:includeSuffixs) {
64+
if (localPath.endsWith(suffix)) {
65+
return "";
66+
}
67+
}
68+
69+
return "do not match include suffix";
70+
}
71+
72+
public String getLocalPath() {
73+
return localPath;
74+
}
75+
76+
public void setLocalPath(String localPath) throws IllegalArgumentException {
77+
File localPathFile = new File(localPath);
78+
if (!localPathFile.exists()) {
79+
throw new IllegalArgumentException("local path not exist!");
80+
}
81+
this.localPath = SystemUtils.formatLocalPath(localPath);
82+
}
83+
84+
public void setExcludes(String excludePath) throws IllegalArgumentException {
85+
excludePath = excludePath.trim();
86+
String[] exludePathArray = excludePath.split(";");
87+
for (String excludePathElement : exludePathArray) {
88+
File tempFile = new File(excludePathElement);
89+
if (!tempFile.exists()) {
90+
throw new IllegalArgumentException("excludePath " + excludePath + " not exist");
91+
}
92+
this.excludes.add(SystemUtils.formatLocalPath(tempFile.getAbsolutePath()));
93+
}
94+
}
95+
96+
public boolean isExcludes(String excludePath) {
97+
return this.excludes.contains(excludePath);
98+
}
99+
100+
public void setIgnoreModifiedTimeLessThan(String ignoreModifiedTimeLessThanStr) {
101+
try {
102+
long number = Long.valueOf(ignoreModifiedTimeLessThanStr);
103+
if (number <= 0) {
104+
throw new IllegalArgumentException(ignoreModifiedTimeLessThanStr + " is invalid, ignoreModifiedTimeLessThan must be positive");
105+
}
106+
this.ignoreModifiedTimeLessThan = number;
107+
} catch (NumberFormatException e) {
108+
throw new IllegalArgumentException("invalid ignoreModifiedTimeLessThan");
109+
}
110+
}
111+
112+
public long getIgnoreModifiedTimeLessThan() {
113+
return ignoreModifiedTimeLessThan;
114+
}
115+
116+
public boolean isFileListMode() {
117+
return fileListMode;
118+
}
119+
120+
public void setFileListMode(boolean fileListMode) {
121+
this.fileListMode = fileListMode;
122+
}
123+
124+
public String getFileListPath() {
125+
return fileListPath;
126+
}
127+
128+
public void setFileListPath(String fileListPath) {
129+
this.fileListPath = fileListPath;
130+
}
131+
132+
public void setCheckLocalRecord(String checkLocalRecord) {
133+
if (checkLocalRecord.compareToIgnoreCase("true") == 0) {
134+
isCheckLocalRecord = true;
135+
} else if (checkLocalRecord.compareToIgnoreCase("false") == 0) {
136+
isCheckLocalRecord = false;
137+
} else {
138+
throw new IllegalArgumentException(checkLocalRecord + " is invalid, checkLocalRecord should be true or false");
139+
}
140+
}
141+
142+
public boolean checkLocalRecord() {
143+
return isCheckLocalRecord;
144+
}
145+
146+
147+
public String toString() {
148+
String strExclude = "";
149+
if (!excludes.isEmpty()) {
150+
StringBuilder sb = new StringBuilder();
151+
for (String element : excludes) {
152+
sb.append(element).append(",");
153+
}
154+
strExclude = sb.toString();
155+
}
156+
157+
158+
String strIgnoreSuffix = "";
159+
if (!ignoreSuffixs.isEmpty()) {
160+
StringBuilder sb = new StringBuilder();
161+
for (String element : ignoreSuffixs) {
162+
sb.append(element).append(",");
163+
}
164+
strIgnoreSuffix = sb.toString();
165+
}
166+
167+
String strIncludeSuffix = "";
168+
if (!includeSuffixs.isEmpty()) {
169+
StringBuilder sb = new StringBuilder();
170+
for (String element : includeSuffixs) {
171+
sb.append(element).append(",");
172+
}
173+
strIncludeSuffix = sb.toString();
174+
}
175+
176+
return super.toString()
177+
+ ". CopyFromLocalConfig:"
178+
+ "localPath=" + getLocalPath()
179+
+ ",excludes=" + strExclude
180+
+ ",ignoreModifiedTimeLessThanSeconds=" + getIgnoreModifiedTimeLessThan()
181+
+ ",ignoreSuffix=" + strIgnoreSuffix
182+
+ ",ignoreEmptyFile=" + ignoreEmptyFile
183+
+ ",includeSuffix=" + strIncludeSuffix
184+
+ ",fileListMode=" + isFileListMode()
185+
+ ",fileListPath=" + getFileListPath()
186+
+ ",checkLocalRecord=" + checkLocalRecord()
187+
;
188+
}
189+
}

‎src/main/java/com/qcloud/cos_migrate_tool/config/MigrateType.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ public enum MigrateType {
88
MIGRATE_FROM_URLLIST("migrateUrl"),
99
MIGRATE_FROM_COS_BUCKET_COPY("migrateBucketCopy"),
1010
MIGRATE_FROM_CSP("migrateCsp"),
11-
MIGRATE_FROM_UPYUN("migrateUpyun");
11+
MIGRATE_FROM_UPYUN("migrateUpyun"),
12+
MIGRATE_FROM_LOCAL_TO_COSN_FS("migrateLocalToCosnFs");
1213

1314
private String migrateType;
1415

0 commit comments

Comments
 (0)