output become more friendly

kitmanzheng · kitmanzheng · commit 27634beacf8f · 2018-11-21T10:04:37.000+08:00
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/meta/TaskStatics.java b/src/main/java/com/qcloud/cos_migrate_tool/meta/TaskStatics.java
@@ -26,7 +26,7 @@ public class TaskStatics {
     private TaskStatics() {}
     
     public void setListFinished(boolean is_finished) {
-        this.list_finished = true;
+        this.list_finished = is_finished;
     }
     
     public boolean getListFinished() {
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateAliTaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateAliTaskExecutor.java
@@ -105,11 +105,8 @@ public void buildTask() {
                                 com.qcloud.cos.utils.UrlEncoderUtils.urlDecode(s.getKey()),
                                 s.getSize(), s.getETag(), smallFileTransferManager,
                                 bigFileTransferManager, recordDb, semaphore);
-                        try {
+
                             AddTask(task);
-                        } catch (InterruptedException e) {
-                            log.error(e.getMessage());
-                        }
                     }
                     nextMarker = com.qcloud.cos.utils.UrlEncoderUtils
                             .urlDecode(objectListing.getNextMarker());
@@ -121,10 +118,22 @@ public void buildTask() {
             } catch (OSSException e) {
                 log.error("list fail msg: {}", e.getMessage());
                 TaskStatics.instance.setListFinished(false);
+                if (e.getErrorCode().equalsIgnoreCase("AccessDenied")) {
+                    System.out.println(e.getMessage());
+                    break;
+                }
             } catch (ClientException e) {
                 log.error("list fail msg: {}", e.getMessage());
                 TaskStatics.instance.setListFinished(false);
+                if (e.getErrorCode().equalsIgnoreCase("AccessDenied")) {
+                    System.out.println(e.getMessage());
+                    break;
+                }
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                TaskStatics.instance.setListFinished(false);
             }
+            retry_num++;
         } while (retry_num < 20);
 
     }
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateAwsTaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateAwsTaskExecutor.java
@@ -106,12 +106,7 @@ public void buildTask() {
                             bigFileTransferManager, recordDb, semaphore);
                     log.info("list key: {}, size: {}, etag: {}", objectSummary.getKey(), objectSummary.getSize(), objectSummary.getETag());
 
-                    try {
                         AddTask(task);
-                    } catch (InterruptedException e) {
-                        log.error(e.getMessage());
-                    }
-
                 }
                 listObjectsRequest.setMarker(objectListing.getNextMarker());
             } while (objectListing.isTruncated());
@@ -125,6 +120,9 @@ public void buildTask() {
         } catch (AmazonClientException ace) {
             log.error("list fail AmazonClientException msg: {}", ace.getMessage().toString());
             TaskStatics.instance.setListFinished(false);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+            TaskStatics.instance.setListFinished(false);     
         }
 
     }
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTask.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTask.java
@@ -91,6 +91,13 @@ public void doTask() {
         }
         CopyObjectRequest copyObjectRequest = new CopyObjectRequest(new Region(srcRegion),
                 srcBucketName, srcKey, destBucketName, destKey);
+        
+        if (srcBucketName.equals(destBucketName) && ( destKey.equals("/" + srcKey) || srcKey.equals(destKey)) ) {
+            ObjectMetadata newObjectMetadata = new ObjectMetadata();
+            newObjectMetadata.addUserMetadata("x-cos-metadata-directive", "Replaced");
+            copyObjectRequest.setNewObjectMetadata(newObjectMetadata);
+        }
+        
         copyObjectRequest.setSourceEndpointSuffix(srcEndpointSuffx);
         try {
             Copy copy = smallFileTransfer.copy(copyObjectRequest, srcCOSClient, null);
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateQiniuTaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateQiniuTaskExecutor.java
@@ -88,11 +88,7 @@ public void buildTask() {
                             item.hash, smallFileTransferManager, bigFileTransferManager, recordDb,
                             semaphore);
 
-                    try {
                         AddTask(task);
-                    } catch (InterruptedException e) {
-                        log.error("add task fail,msg:{}", e.getMessage());
-                    }
                 }
             }
 
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUrllistTask.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUrllistTask.java
@@ -46,8 +46,8 @@ public void doTask() {
             headAttr = Downloader.instance.headFile(url, false);
         } catch (Exception e) {
             String printMsg = String.format("head url attr fail, url: %s", url);
-            System.err.println(printMsg);
-            log.error(printMsg, e);
+            System.err.println(e.toString());
+            log.error(e.toString(), printMsg);
             TaskStatics.instance.addFailCnt();
             return;
         }
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUrllistTaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUrllistTaskExecutor.java
@@ -89,18 +89,19 @@ public FileVisitResult visitFile(Path urllistPath, BasicFileAttributes attrs) th
 
 						MigrateUrllistTask task = new MigrateUrllistTask(config, line, url_path,
 								smallFileTransferManager, bigFileTransferManager, recordDb, semaphore);
-						try {
-							AddTask(task);
-						} catch (InterruptedException e) {
-							log.error("add task fail,msg:{}", e.getMessage());
-						}
+							
+						AddTask(task);
+	
 					}
 					
 					TaskStatics.instance.setListFinished(true);
 					
 				} catch (IOException e) {
 					log.error("read line fail,msg:{}", e.getMessage());
 					TaskStatics.instance.setListFinished(false);
+				} catch (Exception e) {
+				    log.error(e.getMessage());
+				    TaskStatics.instance.setListFinished(false);
 				}
 
 				try {
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/Task.java b/src/main/java/com/qcloud/cos_migrate_tool/task/Task.java
@@ -28,7 +28,7 @@
 public abstract class Task implements Runnable {
     private Semaphore semaphore;
     public static final Logger log = LoggerFactory.getLogger(Task.class);
-
+    protected static Semaphore mutex = new Semaphore(1);
 
     protected TransferManager smallFileTransfer;
     protected TransferManager bigFileTransfer;
@@ -38,6 +38,7 @@ public abstract class Task implements Runnable {
     QUERY_RESULT query_result;
 
 
+
     public Task(Semaphore semaphore, CommonConfig config, TransferManager smallFileTransfer,
             TransferManager bigFileTransfer, RecordDb recordDb) {
         super();
@@ -58,7 +59,7 @@ public boolean isExist(RecordElement recordElement) {
                     recordElement.buildValue());
             return true;
         }
-        
+
         return false;
     }
 
@@ -192,7 +193,7 @@ private String uploadSmallFile(PutObjectRequest putObjectRequest) throws Interru
 
     public String uploadFile(String bucketName, String cosPath, File localFile,
             StorageClass storageClass, boolean entireMd5Attached, ObjectMetadata objectMetadata)
-                    throws Exception {
+            throws Exception {
         PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, cosPath, localFile);
         putObjectRequest.setStorageClass(storageClass);
 
@@ -236,13 +237,21 @@ private void checkTimeWindows() throws InterruptedException {
             if (minuteOfDay >= timeWindowBegin && minuteOfDay <= timeWindowEnd) {
                 return;
             }
-            String printTips =
-                    String.format("currentTime %s, wait next time window [%02d:%02d, %02d:%02d]",
-                            dateTime.toString("yyyy-MM-dd HH:mm:ss"), timeWindowBegin / 60,
-                            timeWindowBegin % 60, timeWindowEnd / 60, timeWindowEnd % 60);
-            System.out.println(printTips);
-            log.info(printTips);
-            Thread.sleep(60000);
+
+            if (mutex.tryAcquire()) {
+                String printTips = String.format(
+                        "currentTime %s, wait next time window [%02d:%02d, %02d:%02d]",
+                        dateTime.toString("yyyy-MM-dd HH:mm:ss"), timeWindowBegin / 60,
+                        timeWindowBegin % 60, timeWindowEnd / 60, timeWindowEnd % 60);
+                System.out.println(printTips);
+                System.out.println(
+                        "---------------------------------------------------------------------");
+                log.info(printTips);
+                Thread.sleep(60000);
+                mutex.release();
+            } else {
+                Thread.sleep(60000);
+            }
         }
     }
 
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java
@@ -36,6 +36,8 @@ public abstract class TaskExecutor {
     protected COSClient cosClient;
     protected TransferManager smallFileTransferManager;
     protected TransferManager bigFileTransferManager;
+
+   
     
     enum RUN_MODE {
         NORMAL, DUMP_REQUESTID, QUERY_REQUESTID;
diff --git a/src/main/java/com/qcloud/cos_migrate_tool/utils/Downloader.java b/src/main/java/com/qcloud/cos_migrate_tool/utils/Downloader.java
@@ -100,8 +100,8 @@ public HeadAttr headFile(String url, boolean qiniuDownLoadFlag) {
                 int http_statuscode = httpResponse.getStatusLine().getStatusCode();
                 if (http_statuscode < 200 || http_statuscode > 299) {
                     String errMsg = String.format(
-                            "head failed, url: %s, httpResponse: %s, response_statuscode: %d", url,
-                            httpResponse.toString(), http_statuscode);
+                            "head failed, response_statuscode: %d, url: %s, httpResponse: %s, ", http_statuscode, url,
+                            httpResponse.toString());
                     throw new Exception(errMsg);
                 }
 

Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ public class TaskStatics {`
`26`	`26`	`private TaskStatics() {}`
`27`	`27`
`28`	`28`	`public void setListFinished(boolean is_finished) {`
`29`		`- this.list_finished = true;`
	`29`	`+ this.list_finished = is_finished;`
`30`	`30`	`}`
`31`	`31`
`32`	`32`	`public boolean getListFinished() {`
Original file line number	Diff line number	Diff line change
`@@ -88,11 +88,7 @@ public void buildTask() {`
`88`	`88`	`item.hash, smallFileTransferManager, bigFileTransferManager, recordDb,`
`89`	`89`	`semaphore);`
`90`	`90`
`91`		`- try {`
`92`	`91`	`AddTask(task);`
`93`		`- } catch (InterruptedException e) {`
`94`		`- log.error("add task fail,msg:{}", e.getMessage());`
`95`		`- }`
`96`	`92`	`}`
`97`	`93`	`}`
`98`	`94`