Merge pull request #24 from jackyding2679/temporaty_token

support temporary token to migrate cos bucket

Author: jackyding2679

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+.project
+db/*
+log/*
+*.iml
+target/

conf/config.ini

@@ -13,6 +13,8 @@ type=migrateLocal
 secretId=AKIDXXXXXXXXXXXXXXXXXXXXXXX
 # 用户的秘钥 secret_key  (可在 https://console.qcloud.com/capi 查看)
 secretKey=yyyyyyyyyyyyyyyyyyyyyyyyyyy
+# 如果使用临时密钥访问存储桶，此处配置临时密钥的Token,该密钥需要有目的桶的PutObject权限(如果迁移类型是migrateBucketCopy，则该密钥需要有源桶的GetObject权限以及目的桶的PutObject权限)
+# token=zzzzzzzzzzzzzzzzz
 # 目的Bucket的名称, 命名规则为{name}-{appid}，即bucket名必须包含appid, 例如movie-1251000000
 bucketName=mybucket-1251668577
 # 目的bucket的region信息. COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
@@ -99,8 +101,6 @@ srcHttps=off
 # 访问ali时, 读取ali响应的超时时间，单位ms, 默认10000
 #srcSocketTimeout=10000
 
-
-
 # 从七牛迁移到COS
 [migrateQiniu]
 # 从七牛迁移到COS的配置分节
@@ -165,8 +165,15 @@ srcEndPointSuffix=
 srcSecretId=xxxxxxxxxxx
 # 源bucket隶属的用户的秘钥 secret_key(可在 https://console.qcloud.com/capi 查看)
 srcSecretKey=yyyyyyyyyyyyyyyy
-# 要迁移的cos路径的 
+# 如果使用临时密钥访问存储桶,此处配置临时密钥的Token,该密钥需要有源桶的GetBucket（列出存储桶对象）和HeadObject权限
+# srcToken=zzzzzzzzzzz
+# 要迁移的cos路径的
 srcCosPath=/
+# 如果只需要迁移部分文件，可以通过srcFileList指定迁移文件列表，例如，只需要迁移aaa/1.jpg和bb/2.jpg，则：
+# $cat /data/fileList.txt
+# aaa/1.jpg
+# bbb/2.jpg
+#srcFileList=/data/fileList.txt
 
 [migrateCsp]
 bucket=xxx

dep/cos_migrate_tool-1.0-jar-with-dependencies.jar

@@ -17,6 +17,7 @@ public class CommonConfig {
     private String region;
     private String ak;
     private String sk;
+    private String token;
     private String cosPath;
     private boolean enableHttps;
     private boolean entireFileMd5Attached;
@@ -157,6 +158,17 @@ public void setSk(String sk) {
         this.sk = sk;
     }
 
+    public String getToken() { 
+        return token;
+    }
+
+    public void setToken(String token) {
+        if (token.trim().isEmpty()) {
+            throw new IllegalArgumentException("token is empty");
+        }
+        this.token = token.trim(); 
+    }
+
     public String getCosPath() {
         return cosPath;
     }

src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java

@@ -34,6 +34,7 @@ public class ConfigParser {
     private static final String COMMON_BUCKETNAME = "bucketName";
     private static final String COMMON_AK = "secretId";
     private static final String COMMON_SK = "secretKey";
+    private static final String COMMON_TOKEN = "token";
     private static final String COMMON_COSPATH = "cosPath";
     private static final String COMMON_DB_COSPATH = "dbCosPath";
     private static final String COMMON_HTTPS = "https";
@@ -94,6 +95,7 @@ public class ConfigParser {
     private static final String COPY_SRC_BUCKETNAME = "srcBucketName";
     private static final String COPY_SRC_SECRETID = "srcSecretId";
     private static final String COPY_SRC_SECRETKEY = "srcSecretKey";
+    private static final String COPY_SRC_TOKEN = "srcToken";
     private static final String COPY_SRC_COSPATH = "srcCosPath";
     private static final String COPY_SRC_ENDPOINT_SUFFIX = "srcEndPointSuffix";
     private static final String COPY_SRC_FILE_LIST = "srcFileList";
@@ -457,7 +459,10 @@ private boolean initCommonConfig(Preferences prefs, CommonConfig commonConfig) {
                 assert (sk != null);
                 commonConfig.setSk(sk);
 
-
+                String token = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_TOKEN);
+                if (token != null) {
+                    commonConfig.setToken(token);
+                }
             }
 
             String cosPathConfig = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_COSPATH);
@@ -893,6 +898,11 @@ private boolean initCopyBucketConfig(Preferences prefs, CopyBucketConfig copyBuc
             assert (srcSecretKey != null);
             copyBucketConfig.setSrcSk(srcSecretKey);
 
+            String token = getConfigValue(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_TOKEN);
+            if (token != null && !token.trim().isEmpty()) {
+                copyBucketConfig.setSrcToken(token);
+            }
+            
             String srcCosPath = getConfigValue(prefs, COPY_BUCKET_SECTION_NAME, COPY_SRC_COSPATH);
             assert (srcCosPath != null);
             copyBucketConfig.setSrcCosPath(srcCosPath);

src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java

@@ -11,6 +11,7 @@ public class CopyBucketConfig extends CommonConfig {
     private String srcBucket;
     private String srcAk;
     private String srcSk;
+    private String srcToken;
     private String srcCosPath;
     private String srcEndpointSuffix;
     private String srcFileList = "";
@@ -74,6 +75,17 @@ public void setSrcSk(String srcSk) {
         this.srcSk = srcSk.trim();
     }
 
+    public String getSrcToken() {
+        return srcToken;
+    }
+
+    public void setSrcToken(String srcToken) {
+        if (srcToken.trim().isEmpty()) {
+            throw new IllegalArgumentException("srcToken is empty");
+        }
+        this.srcToken = srcToken.trim();
+    }
+
     public String getSrcCosPath() {
         return srcCosPath;
     }

src/main/java/com/qcloud/cos_migrate_tool/config/CopyBucketConfig.java

@@ -15,6 +15,7 @@
 import com.qcloud.cos.COSClient;
 import com.qcloud.cos.ClientConfig;
 import com.qcloud.cos.auth.BasicCOSCredentials;
+import com.qcloud.cos.auth.BasicSessionCredentials;
 import com.qcloud.cos.auth.COSCredentials;
 import com.qcloud.cos.http.HttpProtocol;
 import com.qcloud.cos.model.COSObjectSummary;
@@ -38,8 +39,15 @@ public class MigrateCopyBucketTaskExecutor extends TaskExecutor {
 
     public MigrateCopyBucketTaskExecutor(CopyBucketConfig config) {
         super(MigrateType.MIGRATE_FROM_COS_BUCKET_COPY, config);
-
-        COSCredentials srcCred = new BasicCOSCredentials(config.getSrcAk(), config.getSrcSk());
+        String src_token = ((CopyBucketConfig) config).getSrcToken();
+        COSCredentials srcCred = null;
+        if (src_token != null && !src_token.isEmpty()) {
+            log.info("Use temporary token to list object");
+            System.out.println("Use temporary token to list object");
+            srcCred = new BasicSessionCredentials(config.getSrcAk(), config.getSrcSk(), src_token);
+        } else {
+            srcCred = new BasicCOSCredentials(config.getSrcAk(), config.getSrcSk());
+        }
         ClientConfig clientConfig = new ClientConfig(new Region(config.getSrcRegion()));
         if (config.isEnableHttps()) {
             clientConfig.setHttpProtocol(HttpProtocol.https);

src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTaskExecutor.java

@@ -10,6 +10,8 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+import com.qcloud.cos.auth.BasicSessionCredentials;
+import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,7 +59,18 @@ public TaskExecutor(MigrateType migrateType, CommonConfig config) {
         this.smallFileUploadExecutorNum = config.getSmallFileExecutorNumber();
         this.bigFileUploadExecutorNum = config.getBigFileExecutorNum();
 
-        COSCredentials cred = new BasicCOSCredentials(config.getAk(), config.getSk());
+        COSCredentials cred = null;
+        String token = config.getToken();
+        // supporting temporary token
+        if (token != null && !token.isEmpty()) {
+            log.info("Use temporary token to put object");
+            System.out.println("Use temporary token to put object");
+            cred = new BasicSessionCredentials(config.getAk(), config.getSk(), token);
+        } else {
+            cred = new BasicCOSCredentials(config.getAk(), config.getSk());
+        }
+
+        //COSCredentials cred = new BasicCOSCredentials(config.getAk(), config.getSk());
         ClientConfig clientConfig = new ClientConfig(new Region(config.getRegion()));
         if (config.isEnableHttps()) {
             clientConfig.setHttpProtocol(HttpProtocol.https);