Skip to content

Commit b883e7a

Browse files
author
beanwang
committed
support upload file by cosn fs
1 parent 9dba35c commit b883e7a

File tree

13 files changed

+719
-27
lines changed

13 files changed

+719
-27
lines changed

‎conf/config.ini‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,37 @@ fileListPath=/data/config/myFileList.txt
119119
# 这个开关不会控制上传后是否记录数据库, 一定会记录的.
120120
checkLocalRecord=true
121121

122+
# 从本地迁移到COS配置并且使用cosn文件系统分节
123+
[migrateLocalToCosnFs]
124+
# 本地目录, 表示将该路径下的数据都迁移到COS, 对于linux绝对路径, 如/a/b/c, 对于windows绝对路径，注意分隔符为两个反斜杠，如E:\\a\\b\\c
125+
localPath=E:\\code\\java\\workspace\\cos_migrate_tool\\test_data
126+
# 要排除的目录或者文件的绝对路径, 表示将localPath下面某些目录或者文件不进行迁移，多个绝对路径之前用分号分割，不填表示localpath下面的全部迁移
127+
excludes=
128+
# 排除更新时间与当前时间相差不足一定时间段的文件，单位为秒
129+
# 默认不设置, 表示不根据lastmodified时间进行筛选
130+
# 适用于客户在更新文件的同时又在运行迁移工具, 不准备把正在更新的文件迁移上传到COS, 比如设置为300, 表示只上传更新了5分钟以上的文件
131+
ignoreModifiedTimeLessThanSeconds=
132+
# 多个后缀用;隔开，例如:.txt;.tmp;
133+
ignoreSuffix=
134+
# on：忽略空文件，off：不忽略。默认不忽略
135+
ignoreEmptyFile=
136+
137+
# 设定要迁移的文件后缀，不符合后缀的文件会被排除
138+
# 多个后缀用;隔开，例如:.txt;.tmp;
139+
includeSuffix=
140+
# on: 使用列表文件指定所有待迁移文件的相对路径; off: migration递归遍历localPath, 添加待迁移文件; 默认off
141+
# 可使用excludes, ignoreModifiedTimeLessThanSeconds, ignoreSuffix, ignoreEmptyFile忽略列表中的文件
142+
fileListMode=off
143+
# 当fileListMode=on的时候，fileListPath参数为localPath下待迁移文件的相对路径列表
144+
fileListPath=/data/config/myFileList.txt
145+
146+
# 是否检查本地db里有记录. 如果本地db里有记录, 则不再上传此对象. 建议保持默认为 true, 避免迁移时重复上传.
147+
# 如果置为 false, 则不再检查本地db里是否有记录, 即使任务已经执行, 也会尝试重复上传, 甚至可能覆盖服务端数据, 一般不建议使用, 只用于极特殊场景.
148+
# 上传前是否检查服务端有数据，由开关 skipSamePath 控制. 执行顺序是先判断 checkLocalRecord, 再判断 skipSamePath, 其中任何一个判断结果跳过，再跳过不执行上传
149+
# 这个开关不会控制上传后是否记录数据库, 一定会记录的.
150+
checkLocalRecord=true
151+
152+
122153
## 从阿里迁移到COS的配置分节
123154
[migrateAli]
124155
bucket=mybucket-test

‎dep/cos_migrate_tool-1.4.13-jar-with-dependencies.jar‎ renamed to ‎dep/cos_migrate_tool-1.4.13.1-jar-with-dependencies.jar‎

30.8 MB
Binary file not shown.

‎pom.xml‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.qcloud</groupId>
55
<artifactId>cos_migrate_tool</artifactId>
6-
<version>1.4.13</version>
6+
<version>1.4.13.1</version>
77
<packaging>jar</packaging>
88
<properties>
99
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -96,6 +96,31 @@
9696
</exclusion>
9797
</exclusions>
9898
</dependency>
99+
100+
<dependency>
101+
<groupId>com.qcloud.cos</groupId>
102+
<artifactId>hadoop-cos</artifactId>
103+
<version>8.2.7</version>
104+
</dependency>
105+
106+
<dependency>
107+
<groupId>com.qcloud</groupId>
108+
<artifactId>chdfs_hadoop_plugin_network</artifactId>
109+
<version>2.8</version>
110+
</dependency>
111+
112+
<dependency>
113+
<groupId>com.qcloud</groupId>
114+
<artifactId>cos_api-bundle</artifactId>
115+
<version>5.6.112</version>
116+
</dependency>
117+
118+
<dependency>
119+
<groupId>org.apache.hadoop</groupId>
120+
<artifactId>hadoop-common</artifactId>
121+
<version>2.8.5</version>
122+
</dependency>
123+
99124
</dependencies>
100125

101126
<build>

‎src/main/java/com/qcloud/cos_migrate_tool/app/App.java‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,7 @@
77
import java.io.InputStreamReader;
88

99
import com.qcloud.cos.internal.SkipMd5CheckStrategy;
10-
import com.qcloud.cos_migrate_tool.config.CommonConfig;
11-
import com.qcloud.cos_migrate_tool.config.ConfigParser;
12-
import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
13-
import com.qcloud.cos_migrate_tool.config.CopyFromAliConfig;
14-
import com.qcloud.cos_migrate_tool.config.CopyFromAwsConfig;
15-
import com.qcloud.cos_migrate_tool.config.CopyFromCompetitorConfig;
16-
import com.qcloud.cos_migrate_tool.config.CopyFromCspConfig;
17-
import com.qcloud.cos_migrate_tool.config.CopyFromLocalConfig;
18-
import com.qcloud.cos_migrate_tool.config.CopyFromQiniuConfig;
19-
import com.qcloud.cos_migrate_tool.config.CopyFromUpyunConfig;
20-
import com.qcloud.cos_migrate_tool.config.CopyFromUrllistConfig;
21-
import com.qcloud.cos_migrate_tool.config.MigrateType;
10+
import com.qcloud.cos_migrate_tool.config.*;
2211
import com.qcloud.cos_migrate_tool.meta.TaskStatics;
2312
import com.qcloud.cos_migrate_tool.task.MigrateAliTaskExecutor;
2413
import com.qcloud.cos_migrate_tool.task.MigrateAwsTaskExecutor;
@@ -31,6 +20,7 @@
3120
import com.qcloud.cos_migrate_tool.task.MigrateUrllistTaskExecutor;
3221
import com.qcloud.cos_migrate_tool.task.TaskExecutor;
3322

23+
import com.qcloud.cos_migrate_tool.hadoop_fs_task.MigrateLocalToCosnTaskExecutor;
3424
import org.slf4j.Logger;
3525
import org.slf4j.LoggerFactory;
3626

@@ -57,6 +47,8 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
5747
return new MigrateCspTaskExecutor((CopyFromCspConfig) config);
5848
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_UPYUN)) {
5949
return new MigrateUpyunTaskExecutor((CopyFromUpyunConfig) config);
50+
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)) {
51+
return new MigrateLocalToCosnTaskExecutor((CopyFromLocalToCosnConfig) config);
6052
} else {
6153
System.out.println("unknown migrate type");
6254
}

‎src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ public class ConfigParser {
7676
private static final String FILE_LIST_PATH = "fileListPath";
7777
private static final String CHECK_LOCAL_RECORD = "checkLocalRecord";
7878

79+
private static final String LOCAL_TO_COSN_FS_SECTION_NAME = "migrateLocalToCosnFs";
80+
7981
private static final String ALI_SECTION_NAME = "migrateAli";
8082
private static final String AWS_SECTION_NAME = "migrateAws";
8183
private static final String QINIU_SECTION_NAME = "migrateQiniu";
@@ -303,6 +305,14 @@ public boolean parse() {
303305
if (!initCopyFromUpyunConfig(prefs, (CopyFromUpyunConfig) config)) {
304306
return false;
305307
}
308+
} else if (migrateType.equals(MigrateType.MIGRATE_FROM_LOCAL_TO_COSN_FS)){
309+
if (!checkMigrateLocalToCosnFsConfig(prefs)) {
310+
return false;
311+
}
312+
config = new CopyFromLocalToCosnConfig();
313+
if (!initCopyFromLocalToCosnConfig(prefs, (CopyFromLocalToCosnConfig) config)) {
314+
return false;
315+
}
306316
}
307317

308318

@@ -402,6 +412,13 @@ private boolean checkMigrateLocalConfig(Preferences prefs) {
402412
return true;
403413
}
404414

415+
private boolean checkMigrateLocalToCosnFsConfig(Preferences prefs) {
416+
if (!isKeyExist(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH)) {
417+
return false;
418+
}
419+
return true;
420+
}
421+
405422
private boolean checkMigrateCopyBucketConfig(Preferences prefs) {
406423
if (!isKeyExist(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_REGION)) {
407424
return false;
@@ -715,6 +732,69 @@ private boolean initCopyFromLocalConfig(Preferences prefs,
715732
return true;
716733
}
717734

735+
private boolean initCopyFromLocalToCosnConfig(Preferences prefs,
736+
CopyFromLocalToCosnConfig copyLocalConfig) {
737+
if (!initCommonConfig(prefs, copyLocalConfig)) {
738+
return false;
739+
}
740+
try {
741+
742+
String localPathConfig = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_LOCALPATH);
743+
assert (localPathConfig != null);
744+
copyLocalConfig.setLocalPath(localPathConfig);
745+
746+
String excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, LOCAL_EXECLUDE);
747+
if (excludes != null && !excludes.trim().isEmpty()) {
748+
copyLocalConfig.setExcludes(excludes);
749+
} else {
750+
excludes = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, "excludes");
751+
if (excludes != null && !excludes.trim().isEmpty()) {
752+
copyLocalConfig.setExcludes(excludes);
753+
}
754+
}
755+
756+
String ignoreModifiedTimeLessThanStr =
757+
getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_MODIFIED_TIME_LESS_THAN);
758+
if (ignoreModifiedTimeLessThanStr != null
759+
&& !ignoreModifiedTimeLessThanStr.trim().isEmpty()) {
760+
copyLocalConfig.setIgnoreModifiedTimeLessThan(ignoreModifiedTimeLessThanStr);
761+
}
762+
763+
String ignoreSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_SUFFIX);
764+
if (ignoreSuffix != null && !ignoreSuffix.trim().isEmpty()) {
765+
copyLocalConfig.setIgnoreSuffix(ignoreSuffix);
766+
}
767+
768+
String includeSuffix = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, INCLUDE_SUFFIX);
769+
if (includeSuffix != null && !includeSuffix.trim().isEmpty()) {
770+
copyLocalConfig.setIncludeSuffix(includeSuffix);
771+
}
772+
773+
String ignoreEmptyFile = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, IGNORE_EMPTY_FILE);
774+
if (ignoreEmptyFile != null && (ignoreEmptyFile.compareToIgnoreCase("on") == 0)) {
775+
copyLocalConfig.setIgnoreEmptyFile(true);
776+
}
777+
String fileListMode = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_MODE);
778+
if (fileListMode != null && (fileListMode.compareToIgnoreCase("on") == 0)) {
779+
copyLocalConfig.setFileListMode(true);
780+
}
781+
String fileListPath = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, FILE_LIST_PATH);
782+
if (fileListPath != null) {
783+
copyLocalConfig.setFileListPath(fileListPath);
784+
}
785+
786+
String strCheckLocal = getConfigValue(prefs, LOCAL_TO_COSN_FS_SECTION_NAME, CHECK_LOCAL_RECORD);
787+
if (strCheckLocal != null) {
788+
copyLocalConfig.setCheckLocalRecord(strCheckLocal);
789+
}
790+
} catch (Exception e) {
791+
System.err.println(e.getMessage());
792+
log.error(e.getMessage());
793+
return false;
794+
}
795+
return true;
796+
}
797+
718798
private boolean initCopyFromUrllistConfig(Preferences prefs,
719799
CopyFromUrllistConfig copyUrllistConfig) {
720800
if (!initCommonConfig(prefs, copyUrllistConfig)) {

‎src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromLocalConfig.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ public void setIgnoreModifiedTimeLessThan(String ignoreModifiedTimeLessThanStr)
112112
public long getIgnoreModifiedTimeLessThan() {
113113
return ignoreModifiedTimeLessThan;
114114
}
115+
115116
public boolean isFileListMode() {
116117
return fileListMode;
117118
}

0 commit comments

Comments
 (0)