Merge pull request #305 from NigelWu95/dev

gawen957 · web-flow · commit 3f4e9649b20f · 2020-11-23T00:56:59.000+08:00
fix log path set for huawei/tencent client generating, and add upload…
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-version=8.4.5
+version=8.4.6
 
 package_no_test:
 	mvn clean package -Dmaven.test.skip=true
diff --git a/docs/datamigration.md b/docs/datamigration.md
@@ -198,6 +198,9 @@ rs-domain=rs-z0.qiniu.com
 #rs-domain=rs-z1.qiniu.com
 api-domain=api.qiniu.com
 #api-domain=api-z1.qiniu.com
+#上传域名，accup 为加速域名，srcup 为源站域名
+accup-domain=upload.qiniu.com
+srcup-domain=upload.qiniu.com
 #是否使用 https 对上述设置的域名进行请求
 config-https=true/false
 ```  
diff --git a/docs/datasource.md b/docs/datasource.md
@@ -347,6 +347,9 @@ rs-domain=rs-z0.qiniu.com
 #rs-domain=rs-z1.qiniu.com
 api-domain=api.qiniu.com
 #api-domain=api-z1.qiniu.com
+#上传域名，accup 为加速域名，srcup 为源站域名
+accup-domain=upload.qiniu.com
+srcup-domain=upload.qiniu.com
 #是否使用 https 对上述设置的域名进行请求
 config-https=true/false
 ```  
diff --git a/docs/uploadfile.md b/docs/uploadfile.md
@@ -77,6 +77,15 @@ directory-config 中的目录名可以采取相对 path 路径下的目录名。
 |key|上述配置文件中的 "/Users/wubingheng/Projects/Github/test" 等表示目录名或路径，不可重复，重复情况下后者会覆盖前者|  
 |value| 表示数据源中某一行的内容，如 "qiniu_success_1.txt" 表示 "/Users/wubingheng/Projects/Github/test" 目录中可能存在该文件名|  
 
+### 上传域名
+```
+#上传域名，accup 为加速域名，srcup 为源站域名
+accup-domain=upload.qiniu.com
+srcup-domain=upload.qiniu.com
+#是否使用 https 对上述设置的域名进行请求
+config-https=true/false
+```
+
 ### 上传超时时间
 timeout 参数可以通过全局的 timeout 来设置，参考：[网络设置](../README.md#7-网络设置)  
 
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.qiniu</groupId>
     <artifactId>qsuits</artifactId>
-    <version>8.4.5</version>
+    <version>8.4.6</version>
     <name>qsuits</name>
     <description>qiniu-suits is a efficient tools for qiniu api implemented by java8.</description>
     <url>https://github.com/NigelWu95/qiniu-suits-java</url>
diff --git a/src/main/java/com/qiniu/datasource/QiniuLister.java b/src/main/java/com/qiniu/datasource/QiniuLister.java
@@ -220,8 +220,8 @@ public List<FileInfo> currents() {
 
     @Override
     public synchronized String currentEndKey() {
-        if (hasNext()) return CloudApiUtils.decodeQiniuMarker(marker);
-        if (truncateMarker != null && !"".equals(truncateMarker)) return CloudApiUtils.decodeQiniuMarker(truncateMarker);
+        if (hasNext()) return CloudApiUtils.decodeQiniuQosMarker(marker);
+        if (truncateMarker != null && !"".equals(truncateMarker)) return CloudApiUtils.decodeQiniuQosMarker(truncateMarker);
         if (last != null) return last.key;
         if (fileInfoList.size() > 0) last = fileInfoList.get(fileInfoList.size() - 1);
         if (last != null) return last.key;
diff --git a/src/main/java/com/qiniu/datasource/UpLister.java b/src/main/java/com/qiniu/datasource/UpLister.java
@@ -215,9 +215,9 @@ public List<String> getDirectories() {
 
     @Override
     public synchronized String currentEndKey() {
-        if (hasNext()) return CloudApiUtils.decodeUpYunMarker(marker);
+        if (hasNext()) return CloudApiUtils.decodeUpYunYosMarker(marker);
         if (truncateMarker != null && !"".equals(truncateMarker) && !"g2gCZAAEbmV4dGQAA2VvZg".equals(marker)) {
-            return CloudApiUtils.decodeUpYunMarker(truncateMarker);
+            return CloudApiUtils.decodeUpYunYosMarker(truncateMarker);
         }
         if (last != null) return last.key;
         if (fileItems.size() > 0) last = fileItems.get(fileItems.size() - 1);
diff --git a/src/main/java/com/qiniu/datasource/UpYosContainer.java b/src/main/java/com/qiniu/datasource/UpYosContainer.java
@@ -90,7 +90,7 @@ protected IResultOutput getNewResultSaver(String order) throws IOException {
 
     @Override
     protected IStorageLister<FileItem> getLister(String prefix, String marker, String start, String end, int unitLen) throws SuitsException {
-        if (marker == null || "".equals(marker)) marker = CloudApiUtils.getUpYunMarker(username, password, bucket, start);
+        if (marker == null || "".equals(marker)) marker = CloudApiUtils.getUpYunYosMarker(username, password, bucket, start);
         return new UpLister(new UpYunClient(configuration, username, password), bucket, prefix, marker, end, unitLen);
     }
 
diff --git a/src/main/java/com/qiniu/entry/QSuitsEntry.java b/src/main/java/com/qiniu/entry/QSuitsEntry.java
@@ -140,19 +140,23 @@ private Configuration getDefaultQiniuConfig() throws IOException {
     }
 
     private Configuration getDefaultQiniuConfig(String ak, String sk, String regionName, String bucket) throws IOException {
-        com.qiniu.storage.Region region = CloudApiUtils.getQiniuRegion(regionName);
+        com.qiniu.storage.Region region = CloudApiUtils.getQiniuQosRegion(regionName);
         String rsfDomain = entryParam.getValue("rsf-domain", null);
         String rsDomain = entryParam.getValue("rs-domain", null);
         String apiDomain = entryParam.getValue("api-domain", null);
+        String accUpDomain = entryParam.getValue("accup-domain", null);
+        String srcUpDomain = entryParam.getValue("srcup-domain", null);
         if (rsfDomain != null || rsDomain != null || apiDomain != null) {
             com.qiniu.storage.Region.Builder builder = new com.qiniu.storage.Region.Builder();
             if (rsfDomain != null) region = builder.rsfHost(rsfDomain).build();
             if (rsDomain != null) region = builder.rsHost(rsDomain).build();
             if (apiDomain != null) region = builder.apiHost(apiDomain).build();
+            if (apiDomain != null) region = builder.accUpHost(accUpDomain).build();
+            if (apiDomain != null) region = builder.srcUpHost(srcUpDomain).build();
         } else {
             region = (regionName == null || "".equals(regionName)) ?
-                    CloudApiUtils.getQiniuRegion(CloudApiUtils.getQiniuRegion(ak, sk, bucket))
-                    : CloudApiUtils.getQiniuRegion(regionName);
+                    CloudApiUtils.getQiniuQosRegion(CloudApiUtils.getQiniuQosRegion(ak, sk, bucket))
+                    : CloudApiUtils.getQiniuQosRegion(regionName);
         }
         Configuration configuration = new Configuration(region);
         if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
@@ -163,15 +167,19 @@ private Configuration getDefaultQiniuConfig(String ak, String sk, String regionN
     }
 
     private Configuration getNewQiniuConfig() throws IOException {
-        com.qiniu.storage.Region region = CloudApiUtils.getQiniuRegion(regionName);
+        com.qiniu.storage.Region region = CloudApiUtils.getQiniuQosRegion(regionName);
         String rsfDomain = entryParam.getValue("rsf-domain", null);
         String rsDomain = entryParam.getValue("rs-domain", null);
         String apiDomain = entryParam.getValue("api-domain", null);
+        String accUpDomain = entryParam.getValue("accup-domain", null);
+        String srcUpDomain = entryParam.getValue("srcup-domain", null);
         if (rsfDomain != null || rsDomain != null || apiDomain != null) {
             com.qiniu.storage.Region.Builder builder = new com.qiniu.storage.Region.Builder();
             if (rsfDomain != null) region = builder.rsfHost(rsfDomain).build();
             if (rsDomain != null) region = builder.rsHost(rsDomain).build();
             if (apiDomain != null) region = builder.apiHost(apiDomain).build();
+            if (apiDomain != null) region = builder.accUpHost(accUpDomain).build();
+            if (apiDomain != null) region = builder.srcUpHost(srcUpDomain).build();
         }
         Configuration configuration = new Configuration(region);
         if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
diff --git a/src/main/java/com/qiniu/process/huawei/PrivateUrl.java b/src/main/java/com/qiniu/process/huawei/PrivateUrl.java
@@ -6,6 +6,7 @@
 import com.qiniu.interfaces.ILineProcess;
 import com.qiniu.process.Base;
 import com.qiniu.util.CloudApiUtils;
+import com.qiniu.util.LogUtils;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -38,6 +39,7 @@ public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, Str
         configuration.setEndPoint(endpoint);
         obsClient = new ObsClient(accessId, secretKey, configuration);
         CloudApiUtils.checkHuaWei(obsClient);
+        LogUtils.getLogPath(LogUtils.QSUITS);
     }
 
     public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, String endpoint, long expires,
@@ -58,6 +60,7 @@ public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, Str
         configuration.setEndPoint(endpoint);
         obsClient = new ObsClient(accessId, secretKey, configuration);
         CloudApiUtils.checkHuaWei(obsClient);
+        LogUtils.getLogPath(LogUtils.QSUITS);
     }
 
     public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, String endpoint, long expires,
diff --git a/src/main/java/com/qiniu/process/tencent/PrivateUrl.java b/src/main/java/com/qiniu/process/tencent/PrivateUrl.java
@@ -10,6 +10,7 @@
 import com.qiniu.interfaces.ILineProcess;
 import com.qiniu.process.Base;
 import com.qiniu.util.CloudApiUtils;
+import com.qiniu.util.LogUtils;
 
 import java.io.IOException;
 import java.net.URL;
@@ -43,6 +44,7 @@ public PrivateUrl(String secretId, String secretKey, String bucket, String regio
         else clientConfig.setHttpProtocol(HttpProtocol.http);
         cosClient = new COSClient(credentials, clientConfig);
         CloudApiUtils.checkTencent(cosClient);
+        LogUtils.getLogPath(LogUtils.QSUITS);
     }
 
     public PrivateUrl(String secretId, String secretKey, String bucket, String region, boolean useHttps, long expires,
@@ -62,6 +64,7 @@ public PrivateUrl(String secretId, String secretKey, String bucket, String regio
         else clientConfig.setHttpProtocol(HttpProtocol.http);
         cosClient = new COSClient(credentials, clientConfig);
         CloudApiUtils.checkTencent(cosClient);
+        LogUtils.getLogPath(LogUtils.QSUITS);
     }
 
     public PrivateUrl(String secretId, String secretKey, String bucket, String endpoint, boolean useHttps, long expires,
diff --git a/src/main/java/com/qiniu/sdk/UpYunClient.java b/src/main/java/com/qiniu/sdk/UpYunClient.java
@@ -55,7 +55,7 @@ public FileItem getFileInfo(String bucket, String key) throws IOException {
         conn.setUseCaches(false);
         String date = DatetimeUtils.getGMTDate();
         conn.setRequestProperty(UpYunConfig.DATE, date);
-        conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunSign(UpYunConfig.METHOD_HEAD, date, uri,
+        conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunYosSign(UpYunConfig.METHOD_HEAD, date, uri,
                 userName, password, null));
         conn.connect();
         int code = conn.getResponseCode();
@@ -92,7 +92,7 @@ private String HttpGetAction(String uri, Map<String, String> headers) throws IOE
         conn.setUseCaches(false);
         String date = DatetimeUtils.getGMTDate();
         conn.setRequestProperty(UpYunConfig.DATE, date);
-        conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunSign(UpYunConfig.METHOD_GET, date, uri,
+        conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunYosSign(UpYunConfig.METHOD_GET, date, uri,
                 userName, password, null));
         if (headers != null) {
             for (Map.Entry<String, String> entry : headers.entrySet()) {
diff --git a/src/main/java/com/qiniu/util/CloudApiUtils.java b/src/main/java/com/qiniu/util/CloudApiUtils.java
@@ -129,11 +129,7 @@ public static int AliStatusCode(String error, int Default) {
         return aliStatus.getOrDefault(error, Default);
     }
 
-    public static int NetStatusCode(String error, int Default) {
-        return aliStatus.getOrDefault(error, Default);
-    }
-
-    public static String getQiniuMarker(FileInfo fileInfo) {
+    public static String getQiniuQosMarker(FileInfo fileInfo) {
         return getQiniuMarker(fileInfo.key);
     }
 
@@ -153,7 +149,7 @@ public static String getbaiduBosCosMarker(BosObjectSummary obsObject) {
         return obsObject.getKey();
     }
 
-    public static String getUpYunMarker(String bucket, FileItem fileItem) {
+    public static String getUpYunYosMarker(String bucket, FileItem fileItem) {
         if (fileItem.key.contains("/")) {
             String convertedKey = fileItem.key.replace("/", "/~");
             int lastIndex = convertedKey.lastIndexOf("~");
@@ -186,19 +182,19 @@ public static String getTenCosMarker(String key) {
         return key;
     }
 
-    public static String getUpYunMarker(String username, String password, String bucket, String key) throws SuitsException {
+    public static String getUpYunYosMarker(String username, String password, String bucket, String key) throws SuitsException {
         if (key == null || "".equals(key)) return null;
         UpYunClient upYunClient = new UpYunClient(new UpYunConfig(), username, password);
         try {
-            return getUpYunMarker(bucket, upYunClient.getFileInfo(bucket, key));
+            return getUpYunYosMarker(bucket, upYunClient.getFileInfo(bucket, key));
         } catch (Exception e) {
             throw new SuitsException(e, 848);
         } finally {
             upYunClient = null;
         }
     }
 
-    public static String decodeQiniuMarker(String marker) {
+    public static String decodeQiniuQosMarker(String marker) {
         String decodedMarker = new String(Base64.decode(marker, Base64.URL_SAFE | Base64.NO_WRAP));
         JsonObject jsonObject = new JsonParser().parse(decodedMarker).getAsJsonObject();
         return jsonObject.get("k").getAsString();
@@ -216,7 +212,7 @@ public static String decodeHuaweiObsMarker(String marker) {
         return marker;
     }
 
-    public static String decodeUpYunMarker(String marker) {
+    public static String decodeUpYunYosMarker(String marker) {
         String keyString = new String(decoder.decode(marker));
         int index = keyString.contains("/~") ? keyString.indexOf("/~") + 2 : keyString.indexOf("/") + 1;
         return keyString.substring(index).replace("(/~|/@~|/@#)", "/");
@@ -278,7 +274,7 @@ public static void checkBaidu(BosClient bosClient) {
         bosClient.listBuckets();
     }
 
-    public static Region getQiniuRegion(String regionName) throws IOException {
+    public static Region getQiniuQosRegion(String regionName) throws IOException {
         if (regionName == null || "".equals(regionName)) return Region.autoRegion();
         switch (regionName) {
             case "z0":
@@ -305,7 +301,7 @@ public static Region getQiniuRegion(String regionName) throws IOException {
         }
     }
 
-    public static String getQiniuRegion(String accessKey, String secretKey, String bucket) throws SuitsException {
+    public static String getQiniuQosRegion(String accessKey, String secretKey, String bucket) throws SuitsException {
         try {
             Auth auth = Auth.create(accessKey, secretKey);
             Configuration configuration = new Configuration();
@@ -347,6 +343,7 @@ public static String getAliOssRegion(String accessKeyId, String accessKeySecret,
     }
 
     public static String getTenCosRegion(String secretId, String secretKey, String bucket) throws SuitsException {
+        LogUtils.getLogPath(LogUtils.QSUITS);
         COSCredentials cred = new BasicCOSCredentials(secretId, secretKey);
         ClientConfig clientConfig = new ClientConfig();
         COSClient cosClient = new COSClient(cred, clientConfig);
@@ -396,6 +393,7 @@ public static String getS3Region(String s3AccessKeyId, String s3SecretKey, Strin
     }
 
     public static String getHuaweiObsRegion(String accessKeyId, String secretKey, String bucket) throws SuitsException {
+        LogUtils.getLogPath(LogUtils.QSUITS);
         ObsClient obsClient = new ObsClient(accessKeyId, secretKey, "https://obs.myhuaweicloud.com");
         try {
             return obsClient.getBucketLocation(bucket);
@@ -419,7 +417,7 @@ public static String getBaiduBosRegion(String accessKeyId, String secretKey, Str
 
     private static final String lineSeparator = System.getProperty("line.separator");
 
-    public static String upYunSign(String method, String date, String path, String userName, String password, String md5)
+    public static String upYunYosSign(String method, String date, String path, String userName, String password, String md5)
             throws IOException {
         StringBuilder sb = new StringBuilder();
         String sp = "&";
diff --git a/src/main/java/com/qiniu/util/LogUtils.java b/src/main/java/com/qiniu/util/LogUtils.java
@@ -30,7 +30,7 @@ public static String getLogPath(String logName) {
                 } while (logFile.exists());
                 s = String.valueOf(serial);
             }
-            System.setProperty(QSUITS, String.join(FileUtils.pathSeparator, LOG_PATH, QSUITS + s));
+            System.setProperty(logName, String.join(FileUtils.pathSeparator, LOG_PATH, logName + s));
             System.setProperty(PROCEDURE, String.join(FileUtils.pathSeparator, LOG_PATH, PROCEDURE + s));
             return logPath.toString();
         }
diff --git a/src/test/java/com/qiniu/util/CloudApiUtilsTest.java b/src/test/java/com/qiniu/util/CloudApiUtilsTest.java
@@ -11,8 +11,19 @@
 
 public class CloudApiUtilsTest {
 
+    @Test
+    public void testGetQiniuRegion() throws IOException {
+        PropertiesFile propertiesFile = new PropertiesFile("resources/.application.properties");
+        String secretId = propertiesFile.getValue("ak");
+        String secretKey = propertiesFile.getValue("sk");
+        String bucket = propertiesFile.getValue("bucket");
+        String region = CloudApiUtils.getQiniuQosRegion(secretId, secretKey, bucket);
+        System.out.println(bucket + "\t" + region);
+    }
+
     @Test
     public void testGetTenCosRegion() throws IOException {
+        LogUtils.getLogPath(LogUtils.QSUITS);
         PropertiesFile propertiesFile = new PropertiesFile("resources/.tencent.properties");
         String secretId = propertiesFile.getValue("ten-id");
         String secretKey = propertiesFile.getValue("ten-secret");
@@ -21,6 +32,17 @@ public void testGetTenCosRegion() throws IOException {
         System.out.println(bucket + "\t" + region);
     }
 
+    @Test
+    public void testGetHuaweiObsRegion() throws IOException {
+        LogUtils.getLogPath(LogUtils.QSUITS);
+        PropertiesFile propertiesFile = new PropertiesFile("resources/.huawei.properties");
+        String secretId = propertiesFile.getValue("hua-id");
+        String secretKey = propertiesFile.getValue("hua-secret");
+        String bucket = propertiesFile.getValue("bucket");
+        String region = CloudApiUtils.getHuaweiObsRegion(secretId, secretKey, bucket);
+        System.out.println(bucket + "\t" + region);
+    }
+
     @Test
     public void testGetS3Region() throws IOException {
         PropertiesFile propertiesFile = new PropertiesFile("resources/.s3.properties");
@@ -60,19 +82,19 @@ public void testGetUpYunMarker() {
         FileItem fileItem1 = new FileItem();
         fileItem1.key = name1;
         fileItem1.attribute = "folder";
-        System.out.println(CloudApiUtils.getUpYunMarker(bucket, fileItem1));
+        System.out.println(CloudApiUtils.getUpYunYosMarker(bucket, fileItem1));
         String name2 = "wordSplit/xml/20161220/FF8080815919A15101591AFE37C603F7/4028965B591534B501591BBEC0E8049A.txt";
         FileItem fileItem2 = new FileItem();
         fileItem2.key = name2;
-        System.out.println(CloudApiUtils.getUpYunMarker(bucket, fileItem2));
+        System.out.println(CloudApiUtils.getUpYunYosMarker(bucket, fileItem2));
     }
 
     @Test
     public void testDecodeUpYunMarker() {
         String marker1 = "c3F1aXJyZWwvfndvcmRTcGxpdC9+eG1sL34yMDE2MTIyMC9AfkZGODA4MDgxNTkxOUExNTEwMTU5MTlEN0RDOEYwMDM2";
         String marker2 = "c3F1aXJyZWwvfndvcmRTcGxpdC9+eG1sL34yMDE2MTIyMC9+RkY4MDgwODE1OTE5QTE1MTAxNTkxQUZFMzdDNjAzRjcvQCM0MDI4OTY1QjU5MTUzNEI1MDE1OTFCQkVDMEU4MDQ5QS50eHQ=";
-        System.out.println(CloudApiUtils.decodeUpYunMarker(marker1));
-        System.out.println(CloudApiUtils.decodeUpYunMarker(marker2));
+        System.out.println(CloudApiUtils.decodeUpYunYosMarker(marker1));
+        System.out.println(CloudApiUtils.decodeUpYunYosMarker(marker2));
     }
 
     @Test
diff --git a/src/test/java/com/qiniu/util/LogUtilsTest.java b/src/test/java/com/qiniu/util/LogUtilsTest.java
@@ -9,7 +9,10 @@ public class LogUtilsTest {
     @Test
     public void testGetLogPath() {
         System.out.println(LogUtils.getLogPath(LogUtils.QSUITS));
-        System.out.println(LogUtils.getLogPath(LogUtils.ERROR));
+        System.out.println(System.getProperty("qsuits"));
+        System.out.println(System.getProperty("procedure"));
         System.out.println(LogUtils.getLogPath(LogUtils.PROCEDURE));
+        System.out.println(System.getProperty("qsuits"));
+        System.out.println(System.getProperty("procedure"));
     }
 }
diff --git a/version.properties b/version.properties
@@ -1 +1 @@
-version=8.4.5
+version=8.4.6

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-version=8.4.5`
	`1`	`+version=8.4.6`
`2`	`2`
`3`	`3`	`package_no_test:`
`4`	`4`	`mvn clean package -Dmaven.test.skip=true`
Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,7 @@ protected IResultOutput getNewResultSaver(String order) throws IOException {`
`90`	`90`
`91`	`91`	`@Override`
`92`	`92`	`protected IStorageLister<FileItem> getLister(String prefix, String marker, String start, String end, int unitLen) throws SuitsException {`
`93`		`- if (marker == null \|\| "".equals(marker)) marker = CloudApiUtils.getUpYunMarker(username, password, bucket, start);`
	`93`	`+ if (marker == null \|\| "".equals(marker)) marker = CloudApiUtils.getUpYunYosMarker(username, password, bucket, start);`
`94`	`94`	`return new UpLister(new UpYunClient(configuration, username, password), bucket, prefix, marker, end, unitLen);`
`95`	`95`	`}`
`96`	`96`