commit 1.3.0

1. support cas
2. support sse-cos
3. support copy file list

Author: kitmanzheng

dep/cos_migrate_tool-1.0-jar-with-dependencies.jar

@@ -32,7 +32,20 @@ public class CommonConfig {
     private String endpointSuffix = null;
     private String cosProxyHost = "";
     private int cosProxyPort = -1;
+    private String encryptionType = "";
 
+    public void setEncryptionType(String encryptionType) {
+        if (!encryptionType.equals("sse-cos")) {
+            throw new IllegalArgumentException("Not support encryptionType:" + encryptionType);
+        }
+        
+        this.encryptionType = encryptionType.trim();
+    }
+    
+    public String getEncryptionType() {
+        return encryptionType;
+    }
+    
     public String getTempFolderPath() {
         return tempFolderPath;
     }

src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java

@@ -43,6 +43,8 @@ public class ConfigParser {
     private static final String COMMON_EXECUTE_TIME_WINDOW = "executeTimeWindow";
     private static final String COMMON_PROXY_HOST = "proxyHost";
     private static final String COMMON_PROXY_PORT = "proxyPort";
+    private static final String COMMOM_ENCRYPTION_TYPE = "encryptionType";
+    private static final String COMMON_THREAD_NUM = "threadNum";
 
     private static final String LOCAL_SECTION_NAME = "migrateLocal";
     private static final String LOCAL_LOCALPATH = "localPath";
@@ -69,9 +71,12 @@ public class ConfigParser {
     private static final String COPY_SRC_SECRETKEY = "srcSecretKey";
     private static final String COPY_SRC_COSPATH = "srcCosPath";
     private static final String COPY_SRC_ENDPOINT_SUFFIX = "srcEndPointSuffix";
+    private static final String COPY_SRC_FILE_LIST = "srcFileList";
 
     private static final String URLLIST_SECTION_NAME = "migrateUrl";
     private static final String URLLIST_PATH = "urllistPath";
+    
+
 
     private CommonConfig config;
 
@@ -446,6 +451,11 @@ private boolean initCommonConfig(Preferences prefs, CommonConfig commonConfig) {
                 commonConfig.setProxyHost(proxyHost);
             }
 
+            String encryptionType = getConfigValue(prefs, COMMON_SECTION_NAME, COMMOM_ENCRYPTION_TYPE);
+            if (encryptionType!= null && !encryptionType.trim().isEmpty()) {
+                commonConfig.setEncryptionType(encryptionType);
+            }
+            
             int port = -1;
             String portStr = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_PROXY_PORT);
 
@@ -457,6 +467,11 @@ private boolean initCommonConfig(Preferences prefs, CommonConfig commonConfig) {
                     throw new Exception("invalid cos proxy port");
                 }
             }
+            
+            String taskExecutorNumberStr = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_THREAD_NUM);
+            if (taskExecutorNumberStr != null && !taskExecutorNumberStr.isEmpty()) {
+                commonConfig.setTaskExecutorNumberStr(taskExecutorNumberStr);
+            }
 
 
         } catch (Exception e) {
@@ -667,6 +682,11 @@ private boolean initCopyBucketConfig(Preferences prefs, CopyBucketConfig copyBuc
                 copyBucketConfig.setSrcEndpointSuffix(srcEndpointSuffix);
             }
 
+            String fileList = getConfigValue(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_FILE_LIST);
+            if (fileList != null && !fileList.isEmpty()) {
+                copyBucketConfig.setSrcFileList(fileList);
+            }
+                
         } catch (Exception e) {
             System.err.println(e.getMessage());
             log.error(e.getMessage());

src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java

@@ -1,8 +1,10 @@
 package com.qcloud.cos_migrate_tool.config;
 
+import java.io.File;
 import java.util.regex.Pattern;
 
 import com.qcloud.cos_migrate_tool.utils.PathUtils;
+import com.qcloud.cos_migrate_tool.utils.SystemUtils;
 
 public class CopyBucketConfig extends CommonConfig {
     private String srcRegion;
@@ -11,7 +13,20 @@ public class CopyBucketConfig extends CommonConfig {
     private String srcSk;
     private String srcCosPath;
     private String srcEndpointSuffix;
+    private String srcFileList = "";
+    
+    public void setSrcFileList(String srcFileList) {
+        File localPathFile = new File(srcFileList);
+        if (!localPathFile.exists()) {
+            throw new IllegalArgumentException("copy file list:[" + srcFileList + "not exist!");
+        }
+        this.srcFileList = SystemUtils.formatLocalPath(srcFileList);
+    }
 
+    public String getSrcFileList() {
+        return srcFileList;
+    }
+    
     public String getSrcBucket() {
         return srcBucket;
     }

src/main/java/com/qcloud/cos_migrate_tool/config/CopyBucketConfig.java

@@ -5,6 +5,7 @@
 import java.util.concurrent.ThreadLocalRandom;
 
 import com.qcloud.cos.COSClient;
+import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
 import com.qcloud.cos.exception.CosServiceException;
 import com.qcloud.cos.model.CopyObjectRequest;
 import com.qcloud.cos.model.CopyResult;
@@ -27,8 +28,8 @@ public class MigrateCopyBucketTask extends Task {
     private final String srcEndpointSuffx;
     private final String srcBucketName;
     private final String srcKey;
-    private final long srcSize;
-    private final String srcEtag;
+    private  long srcSize;
+    private  String srcEtag;
 
     public MigrateCopyBucketTask(Semaphore semaphore, CopyBucketConfig config,
             TransferManager smallFileTransfer, TransferManager bigFileTransfer, RecordDb recordDb,
@@ -82,6 +83,13 @@ private void transferFileForNotAllowedCopyObject(MigrateCopyBucketRecordElement
 
     @Override
     public void doTask() {
+        
+        if (srcEtag.isEmpty()) {
+            ObjectMetadata objectMetadata = srcCOSClient.getObjectMetadata(srcBucketName, srcKey);
+            srcEtag = objectMetadata.getETag();
+            this.srcSize = objectMetadata.getContentLength();
+        }
+ 
         MigrateCopyBucketRecordElement migrateCopyBucketRecordElement =
                 new MigrateCopyBucketRecordElement(destRegion, destBucketName, destKey, srcRegion,
                         srcBucketName, srcKey, srcSize, srcEtag);
@@ -98,7 +106,13 @@ public void doTask() {
             copyObjectRequest.setNewObjectMetadata(newObjectMetadata);
         }
 
-        copyObjectRequest.setSourceEndpointSuffix(srcEndpointSuffx);
+        if (srcEndpointSuffx != null && !srcEndpointSuffx.isEmpty()) {
+            SuffixEndpointBuilder sourceEndpointBuilder = new SuffixEndpointBuilder(srcEndpointSuffx);
+            copyObjectRequest.setSourceEndpointBuilder(sourceEndpointBuilder);
+        }
+        
+        copyObjectRequest.setStorageClass(this.config.getStorageClass());
+        
         try {
             Copy copy = smallFileTransfer.copy(copyObjectRequest, srcCOSClient, null);
             CopyResult copyResult = copy.waitForCopyResult();

src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTask.java

@@ -1,5 +1,11 @@
 package com.qcloud.cos_migrate_tool.task;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.List;
 
 import org.apache.commons.codec.digest.DigestUtils;
@@ -15,6 +21,7 @@
 import com.qcloud.cos.model.ListObjectsRequest;
 import com.qcloud.cos.model.ObjectListing;
 import com.qcloud.cos.region.Region;
+import com.qcloud.cos.utils.UrlEncoderUtils;
 import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
 import com.qcloud.cos_migrate_tool.config.MigrateType;
 import com.qcloud.cos_migrate_tool.meta.TaskStatics;
@@ -27,6 +34,7 @@ public class MigrateCopyBucketTaskExecutor extends TaskExecutor {
     private String srcRegion;
     private String srcBucketName;
     private String srcCosPath;
+    private String srcFileList;
 
     public MigrateCopyBucketTaskExecutor(CopyBucketConfig config) {
         super(MigrateType.MIGRATE_FROM_COS_BUCKET_COPY, config);
@@ -42,19 +50,20 @@ public MigrateCopyBucketTaskExecutor(CopyBucketConfig config) {
         clientConfig.setConnectionTimeout(5000);
         clientConfig.setSocketTimeout(5000);
 
-        clientConfig.setUserAgent("cos-migrate-tool-v1.0");
+        clientConfig.setUserAgent("cos-migrate-tool-v1.3.0");
         this.srcCosClient = new COSClient(srcCred, clientConfig);
         this.srcRegion = config.getSrcRegion();
         this.srcBucketName = config.getSrcBucket();
         this.srcCosPath = config.getSrcCosPath();
+        this.srcFileList = config.getSrcFileList();
     }
 
     @Override
     protected String buildTaskDbComment() {
         String comment = String.format(
-                "[time: %s], [destRegion: %s], [destBucketName: %s], [destCosFolder: %s], [srcRegion: %s], [srcBucketName: %s], [srcFolder: %s], [smallTaskExecutor: %d]\n",
+                "[time: %s], [destRegion: %s], [destBucketName: %s], [destCosFolder: %s], [srcRegion: %s], [srcFileList:%s], [srcBucketName: %s], [srcFolder: %s], [smallTaskExecutor: %d]\n",
                 SystemUtils.getCurrentDateTime(), config.getRegion(), config.getBucketName(),
-                config.getCosPath(), srcRegion, srcBucketName, srcCosPath,
+                config.getCosPath(), srcRegion, srcFileList, srcBucketName, srcCosPath,
                 this.smallFileUploadExecutorNum);
         return comment;
     }
@@ -73,49 +82,124 @@ protected String buildTaskDbFolderPath() {
 
     @Override
     public void buildTask() {
-        ListObjectsRequest listObjectsRequest =
-                new ListObjectsRequest(srcBucketName, srcCosPath, null, null, 1000);
 
         int lastDelimiter = srcCosPath.lastIndexOf("/");
-        ObjectListing objectListing;
-        int retry_num = 0;
 
-        do {
+        if (!srcFileList.isEmpty()) {
+            File file = new File(srcFileList);
+            if (!file.isFile() || !file.exists()) {
+                String printMsg = String.format("file[%s] not exist or not file", srcFileList);
+                log.error(printMsg);
+                System.out.println(printMsg);
+            }
+
+            InputStreamReader read = null;
             try {
-                while (true) {
-                    objectListing = srcCosClient.listObjects(listObjectsRequest);
-                    List<COSObjectSummary> cosObjectSummaries = objectListing.getObjectSummaries();
-                    for (COSObjectSummary cosObjectSummary : cosObjectSummaries) {
-                        String srcKey = cosObjectSummary.getKey();
-                        String srcEtag = cosObjectSummary.getETag();
-                        long srcSize = cosObjectSummary.getSize();
-                        String keyName = srcKey.substring(lastDelimiter);
-                        String copyDestKey = config.getCosPath() + keyName;
-           
-                        MigrateCopyBucketTask task =
-                                new MigrateCopyBucketTask(semaphore, (CopyBucketConfig) config,
-                                        smallFileTransferManager, bigFileTransferManager, recordDb,
-                                        srcCosClient, srcKey, srcSize, srcEtag, copyDestKey);
-                        AddTask(task);
-                    }
-                    if (!objectListing.isTruncated()) {
-                        break;
+                read = new InputStreamReader(new FileInputStream(file));
+            } catch (FileNotFoundException e1) {
+                e1.printStackTrace();
+                return;
+            }
+            
+            BufferedReader bufferedReader = new BufferedReader(read);
+            String srcKey = null;
+
+            try {
+                while ((srcKey = bufferedReader.readLine()) != null){
+                    
+                    srcKey = UrlEncoderUtils.urlDecode(srcKey);
+
+                    String copyDestKey = null;
+                    if (srcKey.startsWith("/")) {
+                        copyDestKey = config.getCosPath() + srcKey.substring(1);
+                        srcKey = srcKey.substring(1);
+                    } else {
+                        copyDestKey = config.getCosPath() + srcKey;
                     }
-                    listObjectsRequest.setMarker(objectListing.getNextMarker());
+                    
+                    MigrateCopyBucketTask task = new MigrateCopyBucketTask(semaphore,
+                            (CopyBucketConfig) config, smallFileTransferManager,
+                            bigFileTransferManager, recordDb, srcCosClient, srcKey, 0,
+                            "", copyDestKey);
+                    
+                        AddTask(task);
+    
                 }
-
+                
                 TaskStatics.instance.setListFinished(true);
 
-                return;
-
+            } catch (IOException e) {
+                log.error(e.toString());
+                TaskStatics.instance.setListFinished(false);
+                e.printStackTrace();
             } catch (Exception e) {
-                log.error("List cos bucket occur a exception", e);
+                log.error(e.toString());
+                e.printStackTrace();
                 TaskStatics.instance.setListFinished(false);
             }
 
-            ++retry_num;
-            
-        } while (retry_num < 20);
+            try {
+                bufferedReader.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            try {
+                read.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+      
+
+        } else {
+
+            ListObjectsRequest listObjectsRequest =
+                    new ListObjectsRequest(srcBucketName, srcCosPath, null, null, 1000);
+
+            ObjectListing objectListing;
+            int retry_num = 0;
+
+            do {
+                try {
+                    while (true) {
+                        objectListing = srcCosClient.listObjects(listObjectsRequest);
+                        List<COSObjectSummary> cosObjectSummaries =
+                                objectListing.getObjectSummaries();
+                        
+                        for (COSObjectSummary cosObjectSummary : cosObjectSummaries) {
+                            String srcKey = cosObjectSummary.getKey();
+                            String srcEtag = cosObjectSummary.getETag();
+                            long srcSize = cosObjectSummary.getSize();
+                            String keyName = srcKey.substring(lastDelimiter);
+                            String copyDestKey = config.getCosPath() + keyName;
+                          
+                            MigrateCopyBucketTask task = new MigrateCopyBucketTask(semaphore,
+                                    (CopyBucketConfig) config, smallFileTransferManager,
+                                    bigFileTransferManager, recordDb, srcCosClient, srcKey, srcSize,
+                                    srcEtag, copyDestKey);
+                            
+                            AddTask(task);
+                        }
+                        
+                        if (!objectListing.isTruncated()) {
+                            break;
+                        }
+                        
+                        listObjectsRequest.setMarker(objectListing.getNextMarker());
+                    }
+
+                    TaskStatics.instance.setListFinished(true);
+
+                    return;
+
+                } catch (Exception e) {
+                    log.error("List cos bucket occur a exception:{}", e.toString());
+                    TaskStatics.instance.setListFinished(false);
+                }
+
+                ++retry_num;
+
+            } while (retry_num < 20);
+        }
 
     }
 

src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTaskExecutor.java

@@ -202,6 +202,10 @@ public String uploadFile(String bucketName, String cosPath, File localFile,
             objectMetadata.addUserMetadata("md5", md5);
         }
 
+        if (config.getEncryptionType().equals("sse-cos")) {
+            objectMetadata.setServerSideEncryption("AES256");
+        }
+        
         putObjectRequest.setMetadata(objectMetadata);
         int retryTime = 0;
         final int maxRetry = 5;

src/main/java/com/qcloud/cos_migrate_tool/task/Task.java

@@ -55,10 +55,11 @@ public TaskExecutor(MigrateType migrateType, CommonConfig config) {
         if (config.isEnableHttps()) {
             clientConfig.setHttpProtocol(HttpProtocol.https);
         }
+        
         if (config.getEndpointSuffix() != null) {
             clientConfig.setEndPointSuffix(config.getEndpointSuffix());
         }
-        clientConfig.setUserAgent("cos-migrate-tool-v1.2.5");
+        clientConfig.setUserAgent("cos-migrate-tool-v1.3.0");
 
         if (!config.getProxyHost().isEmpty() && config.getProxyPort() > 0) {
             clientConfig.setHttpProxyIp(config.getProxyHost());