Skip to content

Commit 1d667e6

Browse files
kitmanzhengkitmanzheng
committed
update upyun
1. add retry 2. compare md5
1 parent 37a3264 commit 1d667e6

File tree

6 files changed

+170
-118
lines changed

6 files changed

+170
-118
lines changed

‎dep/cos_migrate_tool-1.0-jar-with-dependencies.jar‎

962 Bytes
Binary file not shown.

‎src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ public class ConfigParser {
7474
private static final String OSS_PROXY_HOST = "proxyHost";
7575
private static final String OSS_PROXY_PORT = "proxyPort";
7676
private static final String OSS_URL_LIST = "uriList";
77+
private static final String UPYUN_COMPARE_MD5 = "compareMd5";
7778

7879
private static final String QINIU_NEED_SIGN = "needSign";
7980

@@ -690,7 +691,23 @@ private boolean initCopyFromUpyunConfig(Preferences prefs, CopyFromUpyunConfig c
690691
if (!initCopyFromCompetitorConfig(prefs, copyUpyunConfig)) {
691692
return false;
692693
}
693-
694+
695+
696+
697+
String compareMd5 = getConfigValue(prefs, UPYUN_SECTION_NAME, UPYUN_COMPARE_MD5);
698+
if (compareMd5 != null) {
699+
if (compareMd5.compareToIgnoreCase("off") == 0) {
700+
copyUpyunConfig.setCompareMd5(false);
701+
} else if (compareMd5.compareToIgnoreCase("on") == 0) {
702+
copyUpyunConfig.setCompareMd5(true);
703+
} else {
704+
String errMsg = "upyun section compareMd5 invalid,need to be \"on\" or \"off\".\n";
705+
System.err.println(errMsg);
706+
log.error(errMsg);
707+
return false;
708+
}
709+
}
710+
694711
return true;
695712
}
696713

‎src/main/java/com/qcloud/cos_migrate_tool/config/CopyFromUpyunConfig.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,13 @@
22

33

44
public class CopyFromUpyunConfig extends CopyFromCompetitorConfig {
5-
5+
private boolean compareMd5 = true;
6+
7+
public boolean isCompareMd5() {
8+
return compareMd5;
9+
}
10+
11+
public void setCompareMd5(boolean compareMd5) {
12+
this.compareMd5 = compareMd5;
13+
}
614
}

‎src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUpyunTask.java‎

Lines changed: 87 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,23 @@ public class MigrateUpyunTask extends Task {
3636
private String contentType;
3737

3838
public MigrateUpyunTask(CopyFromUpyunConfig config, UpYun upyun, String srcKey, long fileSize,
39-
Date lastModify, String contentType, TransferManager smallFileTransfer, TransferManager bigFileTransfer,
40-
RecordDb recordDb, Semaphore semaphore) {
39+
Date lastModify, String contentType, TransferManager smallFileTransfer,
40+
TransferManager bigFileTransfer, RecordDb recordDb, Semaphore semaphore) {
4141
super(semaphore, config, smallFileTransfer, bigFileTransfer, recordDb);
42-
this.upyun = upyun;
42+
//this.upyun = upyun;
43+
44+
//又拍云sdk多线程有坑，headers不对
45+
this.upyun = new UpYun(config.getSrcBucket(), config.getSrcAccessKeyId(), config.getSrcAccessKeySecret());
46+
this.upyun.setTimeout(60);
47+
this.upyun.setApiDomain(UpYun.ED_AUTO);
48+
49+
if (!config.getSrcProxyHost().isEmpty() && config.getSrcProxyPort() > 0) {
50+
System.setProperty("java.net.useSystemProxies", "true");
51+
System.setProperty("http.proxyHost", config.getSrcProxyHost());
52+
System.setProperty("http.proxyPort", Integer.toString(config.getSrcProxyPort()));
53+
}
54+
55+
4356
this.srcKey = srcKey;
4457
this.fileSize = fileSize;
4558
this.contentType = contentType;
@@ -73,7 +86,6 @@ public void doTask() {
7386
String cosPath = buildCOSPath();
7487

7588
this.etag = this.lastModify.toString();
76-
// System.out.println(this.etag);
7789

7890
MigrateCompetitorRecordElement upyunRecordElement = new MigrateCompetitorRecordElement(
7991
MigrateType.MIGRATE_FROM_UPYUN, config.getBucketName(), cosPath, etag, fileSize);
@@ -118,30 +130,73 @@ public void doTask() {
118130

119131
String localPath = config.getTempFolderPath() + UUID.randomUUID().toString();
120132
File localFile = new File(localPath);
121-
try {
122-
boolean success =
123-
upyun.readFile(UrlEncoderUtils.encodeEscapeDelimiter(this.srcKey), localFile);
124-
if (!success) {
125-
String errMsg = String.format("[fail] taskInfo: %s, No Exception",
126-
upyunRecordElement.buildKey());
127-
System.err.println(errMsg);
128-
log.error(errMsg);
129-
TaskStatics.instance.addFailCnt();
133+
int retry_limit = 5;
134+
boolean download_success = false;
135+
String contentMd5 = "";
136+
137+
do {
138+
try {
139+
140+
if (((CopyFromUpyunConfig) config).isCompareMd5()) {
141+
Map<String, String> headers = this.upyun.getFileInfo(UrlEncoderUtils.encodeEscapeDelimiter(this.srcKey));
142+
if (headers == null || !headers.containsKey("Content-MD5")) {
143+
String errMsg = String
144+
.format("[fail] taskInfo: %s, can't get fileinfo or content-md5", upyunRecordElement.buildKey());
145+
System.err.println(errMsg);
146+
log.error(errMsg);
147+
TaskStatics.instance.addFailCnt();
148+
return;
149+
}
150+
151+
contentMd5 = headers.get("Content-MD5");
152+
}
153+
154+
download_success = this.upyun
155+
.readFile(UrlEncoderUtils.encodeEscapeDelimiter(this.srcKey), localFile);
156+
if (!download_success) {
157+
String errMsg = String.format("[fail] taskInfo: %s, No Exception",
158+
upyunRecordElement.buildKey());
159+
System.err.println(errMsg);
160+
log.error(errMsg);
161+
TaskStatics.instance.addFailCnt();
162+
if (localFile.exists()) {
163+
localFile.delete();
164+
}
165+
}
166+
167+
168+
} catch (Exception e) {
169+
download_success = false;
170+
retry_limit--;
171+
172+
try {
173+
Thread.sleep(300);
174+
} catch (InterruptedException e1) {
175+
// TODO Auto-generated catch block
176+
e1.printStackTrace();
177+
}
178+
179+
if (retry_limit == 0) {
180+
String errMsg =
181+
String.format("[fail] taskInfo: %s, Caught an Exception, error msg: %s",
182+
upyunRecordElement.buildKey(), e.toString());
183+
System.err.println(errMsg);
184+
log.error(errMsg);
185+
TaskStatics.instance.addFailCnt();
186+
if (localFile.exists()) {
187+
localFile.delete();
188+
}
189+
return;
190+
}
191+
130192
if (localFile.exists()) {
131193
localFile.delete();
132194
}
195+
133196
}
134-
} catch (Exception e) {
135-
String errMsg = String.format("[fail] taskInfo: %s, Caught an Exception, error msg: %s",
136-
upyunRecordElement.buildKey(), e.toString());
137-
System.err.println(errMsg);
138-
log.error(errMsg);
139-
TaskStatics.instance.addFailCnt();
140-
if (localFile.exists()) {
141-
localFile.delete();
142-
}
143-
return;
144-
}
197+
198+
199+
} while (!download_success && retry_limit > 0);
145200

146201
// upload
147202

@@ -166,8 +221,12 @@ public void doTask() {
166221
}
167222

168223
com.qcloud.cos.model.ObjectMetadata cosMetadata = new com.qcloud.cos.model.ObjectMetadata();
169-
cosMetadata.setContentType(this.contentType);
170-
224+
if (((CopyFromUpyunConfig) config).isCompareMd5()) {
225+
cosMetadata.addUserMetadata("upyun-etag", contentMd5);
226+
}
227+
228+
cosMetadata.setContentType(contentType);
229+
171230
try {
172231
String requestId = uploadFile(config.getBucketName(), cosPath, localFile,
173232
config.getStorageClass(), config.isEntireFileMd5Attached(), cosMetadata, null);
@@ -183,10 +242,9 @@ public void doTask() {
183242
System.out.println(printMsg);
184243
log.info(printMsg);
185244
} catch (Exception e) {
186-
String printMsg = String.format("[fail] task_info: %s", upyunRecordElement.buildKey());
245+
String printMsg = String.format("[fail] task_info: %s exception: %s", upyunRecordElement.buildKey(), e.toString());
187246
System.err.println(printMsg);
188-
log.error("[fail] task_info: {}, exception: {}", upyunRecordElement.buildKey(),
189-
e.toString());
247+
log.error(printMsg);
190248
TaskStatics.instance.addFailCnt();
191249
} finally {
192250
localFile.delete();

‎src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUpyunTaskExecutor.java‎

Lines changed: 45 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public MigrateUpyunTaskExecutor(CopyFromUpyunConfig config) {
5151
this.upyun = new UpYun(this.srcBucket, this.srcAccessKeyId, this.srcAccessKeySecret);
5252
upyun.setTimeout(60);
5353
upyun.setApiDomain(UpYun.ED_AUTO);
54-
54+
5555
if (!config.getSrcProxyHost().isEmpty() && config.getSrcProxyPort() > 0) {
5656
System.out.println("use proxy");
5757
System.setProperty("java.net.useSystemProxies", "true");
@@ -80,102 +80,60 @@ public String buildTaskDbFolderPath() {
8080

8181
public void buildTask() {
8282

83-
try {
84-
LinkedList<String> dirList = new LinkedList<String>();
85-
dirList.add("");
86-
while (!dirList.isEmpty()) {
87-
String curDir = dirList.removeFirst();
88-
String lastItr = "";
89-
90-
FolderItemIter folderItemIter;
91-
do {
92-
Map<String, String> params = new HashMap<String, String>();
93-
94-
params.put("x-list-iter", lastItr);
95-
params.put("x-list-limit", "1000");
96-
97-
folderItemIter = upyun.readDirIter(curDir, params);
98-
lastItr = folderItemIter.iter;
99-
for (int i = 0; i < folderItemIter.files.size(); ++i) {
100-
if (folderItemIter.files.get(i).type.equals("folder")) {
101-
102-
dirList.add(curDir + "/" + folderItemIter.files.get(i).name);
103-
} else {
104-
MigrateUpyunTask task = new MigrateUpyunTask(config, upyun,
105-
curDir + "/" + folderItemIter.files.get(i).name,
106-
folderItemIter.files.get(i).size,
107-
folderItemIter.files.get(i).date, folderItemIter.files.get(i).type, smallFileTransferManager,
108-
bigFileTransferManager, recordDb, semaphore);
109-
110-
AddTask(task);
111-
}
112-
}
113-
} while (folderItemIter.files.size() > 0);
114-
}
115-
116-
TaskStatics.instance.setListFinished(true);
117-
118-
} catch (Exception e) {
119-
log.error(e.getMessage());
120-
TaskStatics.instance.setListFinished(false);
121-
}
122-
123-
124-
125-
/*
126-
final int maxKeys = 200;
127-
final String keyPrefix = this.srcPrefix;
128-
String nextMarker = "";
129-
ObjectListing objectListing;
130-
13183
int retry_num = 0;
84+
LinkedList<String> dirList = new LinkedList<String>();
85+
dirList.add("");
13286

13387
do {
13488
try {
135-
do {
136-
objectListing = ossClient.listObjects(new ListObjectsRequest(this.srcBucket)
137-
.withPrefix(keyPrefix).withMarker(nextMarker).withMaxKeys(maxKeys)
138-
.withEncodingType("url"));
139-
log.info("list next marker: " + nextMarker);
140-
List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
141-
for (OSSObjectSummary s : sums) {
142-
// AddTask
143-
MigrateAliTask task = new MigrateAliTask(config, ossClient,
144-
com.qcloud.cos.utils.UrlEncoderUtils.urlDecode(s.getKey()),
145-
s.getSize(), s.getETag(), s.getLastModified(), smallFileTransferManager,
146-
bigFileTransferManager, recordDb, semaphore);
147-
148-
AddTask(task);
149-
}
150-
nextMarker = com.qcloud.cos.utils.UrlEncoderUtils
151-
.urlDecode(objectListing.getNextMarker());
152-
} while (objectListing.isTruncated());
153-
89+
while (!dirList.isEmpty()) {
90+
String curDir = dirList.removeFirst();
91+
String lastItr = "";
92+
93+
FolderItemIter folderItemIter;
94+
do {
95+
Map<String, String> params = new HashMap<String, String>();
96+
97+
params.put("x-list-iter", lastItr);
98+
params.put("x-list-limit", "1000");
99+
100+
folderItemIter = upyun.readDirIter(curDir, params);
101+
lastItr = folderItemIter.iter;
102+
for (int i = 0; i < folderItemIter.files.size(); ++i) {
103+
if (folderItemIter.files.get(i).type.equals("folder")) {
104+
105+
dirList.add(curDir + "/" + folderItemIter.files.get(i).name);
106+
} else {
107+
MigrateUpyunTask task = new MigrateUpyunTask(config, null,
108+
curDir + "/" + folderItemIter.files.get(i).name,
109+
folderItemIter.files.get(i).size,
110+
folderItemIter.files.get(i).date,
111+
folderItemIter.files.get(i).type, smallFileTransferManager,
112+
bigFileTransferManager, recordDb, semaphore);
113+
114+
AddTask(task);
115+
}
116+
}
117+
} while (folderItemIter.files.size() > 0);
118+
}
119+
154120
TaskStatics.instance.setListFinished(true);
155121
return;
156-
157-
} catch (OSSException e) {
158-
log.error("list fail msg: {}", e.getMessage());
159-
TaskStatics.instance.setListFinished(false);
160-
if (e.getErrorCode().equalsIgnoreCase("AccessDenied")) {
161-
System.out.println(e.getMessage());
162-
break;
163-
}
164-
} catch (ClientException e) {
165-
log.error("list fail msg: {}", e.getMessage());
166-
TaskStatics.instance.setListFinished(false);
167-
if (e.getErrorCode().equalsIgnoreCase("AccessDenied")) {
168-
System.out.println(e.getMessage());
169-
break;
170-
}
122+
171123
} catch (Exception e) {
172-
log.error(e.getMessage());
124+
log.error("retry_time:{}, Exception:{}", retry_num,e.getMessage());
173125
TaskStatics.instance.setListFinished(false);
174126
}
127+
128+
try {
129+
Thread.sleep(200);
130+
} catch (InterruptedException e) {
131+
// TODO Auto-generated catch block
132+
e.printStackTrace();
133+
}
134+
175135
retry_num++;
176-
} while (retry_num < 20);
177-
178-
*/
136+
} while (retry_num < 300);
179137

180138
}
181139

‎src/main/java/com/qcloud/cos_migrate_tool/task/Task.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import com.qcloud.cos.transfer.Upload;
2323
import com.qcloud.cos.utils.Md5Utils;
2424
import com.qcloud.cos_migrate_tool.config.CommonConfig;
25+
import com.qcloud.cos_migrate_tool.config.ConfigParser;
26+
import com.qcloud.cos_migrate_tool.config.MigrateType;
2527
import com.qcloud.cos_migrate_tool.record.RecordDb;
2628
import com.qcloud.cos_migrate_tool.record.RecordDb.QUERY_RESULT;
2729
import com.qcloud.cos_migrate_tool.record.RecordElement;
@@ -213,6 +215,15 @@ public String uploadFile(String bucketName, String cosPath, File localFile,
213215

214216
if (entireMd5Attached) {
215217
String md5 = Md5Utils.md5Hex(localFile);
218+
219+
String upyunTag = objectMetadata.getUserMetaDataOf("upyun-etag");
220+
if (upyunTag != null) {
221+
if (!md5.equalsIgnoreCase(upyunTag)) {
222+
String exceptionMsg = String.format("md5 is not match upyun[%s] local[%s]",
223+
upyunTag, md5);
224+
throw new Exception(exceptionMsg);
225+
}
226+
}
216227
objectMetadata.addUserMetadata("md5", md5);
217228
}
218229

0 commit comments

Comments
 (0)