optimize download (#401)

Author: YangSen-qn

cmd_test/async_fetch_test.go

@@ -4,6 +4,8 @@ package cmd
 
 import (
 	"github.com/qiniu/qshell/v2/cmd_test/test"
+	"github.com/qiniu/qshell/v2/iqshell/common/utils"
+
 	"strings"
 	"testing"
 )
@@ -15,7 +17,7 @@ func TestAsyncFetch(t *testing.T) {
 	fetchKeys = append(fetchKeys, "hello10.json")
 	content := ""
 	for _, key := range fetchKeys {
-		content += "http://" + test.BucketDomain + "/" + key + "\t" + "0" + "\t" + "fetch_" + key + "\n"
+		content += utils.Endpoint(false, test.BucketDomain) + "/" + key + "\t" + "0" + "\t" + "fetch_" + key + "\n"
 	}
 	path, err := test.CreateFileWithContent("async_fetch.txt", content)
 	if err != nil {

cmd_test/download_get_test.go

@@ -3,10 +3,12 @@
 package cmd
 
 import (
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
+	"github.com/qiniu/qshell/v2/iqshell/common/utils"
 )
 
 func TestGetImage(t *testing.T) {
@@ -130,7 +132,7 @@ func TestGetWithDomain(t *testing.T) {
 	}
 	path := filepath.Join(resultPath, test.Key)
 	_, errs := test.RunCmdWithError("get", test.Bucket, test.Key,
-		"--domain", test.BucketDomain,
+		"--domain", utils.Endpoint(false, test.BucketDomain),
 		"-o", path)
 	defer test.RemoveFile(path)
 
@@ -142,6 +144,27 @@ func TestGetWithDomain(t *testing.T) {
 	}
 }
 
+func TestGetWithErrorDomain(t *testing.T) {
+	TestCopy(t)
+
+	resultPath, err := test.ResultPath()
+	if err != nil {
+		t.Fatal("get result path error:", err)
+	}
+	path := filepath.Join(resultPath, test.Key)
+	_, errs := test.RunCmdWithError("get", test.Bucket, test.Key,
+		"--domain", "error.qiniu.com",
+		"-o", path)
+	defer test.RemoveFile(path)
+
+	if len(errs) == 0 {
+		t.Fail()
+	}
+	if test.IsFileHasContent(path) {
+		t.Fatal("TestGetWithErrorDomain get file content should be empty")
+	}
+}
+
 func TestGetNoExistDomain(t *testing.T) {
 	resultPath, err := test.ResultPath()
 	if err != nil {

cmd_test/download_test.go

@@ -5,13 +5,15 @@ package cmd
 import (
 	"encoding/json"
 	"errors"
+	"path/filepath"
+	"strings"
+	"testing"
+
 	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"github.com/qiniu/qshell/v2/iqshell/common/config"
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
+	"github.com/qiniu/qshell/v2/iqshell/common/utils"
 	"github.com/qiniu/qshell/v2/iqshell/storage/object/download/operations"
-	"path/filepath"
-	"strings"
-	"testing"
 )
 
 func TestDownloadWithKeyFile(t *testing.T) {
@@ -70,7 +72,7 @@ func TestDownloadFromBucket(t *testing.T) {
 			Bucket:          test.Bucket,
 			Prefix:          "hello3,hello5,hello7",
 			Suffixes:        "",
-			IoHost:          test.BucketDomain,
+			IoHost:          utils.Endpoint(false, test.BucketDomain),
 			Public:          true,
 			CheckSize:       true,
 			Referer:         "",
@@ -111,11 +113,11 @@ func TestDownloadWithDomain(t *testing.T) {
 			Bucket:          test.Bucket,
 			Prefix:          "hello3,hello5,hello7",
 			Suffixes:        "",
-			IoHost:          test.BucketDomain,
+			IoHost:          utils.Endpoint(false, test.BucketDomain),
 			Public:          true,
 			CheckSize:       true,
 			Referer:         "",
-			Domain:          test.BucketDomain,
+			Domain:          utils.Endpoint(false, test.BucketDomain),
 			RecordRoot:      "",
 		},
 	}
@@ -151,7 +153,7 @@ func TestDownloadNoBucket(t *testing.T) {
 			Bucket:     "",
 			Prefix:     "hello3,hello5,hello7",
 			Suffixes:   "",
-			IoHost:     test.BucketDomain,
+			IoHost:     utils.Endpoint(false, test.BucketDomain),
 			Public:     true,
 			CheckHash:  true,
 			Referer:    "",
@@ -438,7 +440,7 @@ func TestDownload2PublicWithDomain(t *testing.T) {
 
 	test.RunCmdWithError("qdownload2",
 		"--bucket", test.Bucket,
-		"--domain", test.BucketDomain,
+		"--domain", utils.Endpoint(false, test.BucketDomain),
 		"--dest-dir", destDir,
 		"--key-file", keysFilePath,
 		"--log-file", logPath,
@@ -462,6 +464,42 @@ func TestDownload2PublicWithDomain(t *testing.T) {
 	return
 }
 
+func TestDownload2PublicWithErrorDomain(t *testing.T) {
+	test.RemoveRootPath()
+
+	keys := test.KeysString + "\nhello_10.json"
+	keysFilePath, err := test.CreateFileWithContent("download_keys.txt", keys)
+	if err != nil {
+		t.Fatal("create cdn config file error:", err)
+	}
+
+	rootPath, err := test.RootPath()
+	if err != nil {
+		t.Fatal("get root path error:", err)
+	}
+
+	destDir := filepath.Join(rootPath, "download2")
+
+	errString, _ := test.RunCmdWithError("qdownload2",
+		"--bucket", test.Bucket,
+		"--domain", "error.qiniu.com",
+		"--dest-dir", destDir,
+		"--key-file", keysFilePath,
+		"--log-level", "debug",
+		"--public",
+		"-c", "4",
+		"-d")
+	if len(errString) == 0 {
+		t.Fatal("should be error")
+	}
+
+	if test.FileCountInDir(destDir) > 0 {
+		t.Fatal("no file should be download")
+	}
+
+	return
+}
+
 func TestDownload2Document(t *testing.T) {
 	test.TestDocument("qdownload2", t)
 }

cmd_test/fop_test.go

@@ -3,9 +3,10 @@
 package cmd
 
 import (
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 const fopObjectKey = "test_mv.mp4"
@@ -19,7 +20,7 @@ func TestFop(t *testing.T) {
 
 	result = strings.ReplaceAll(result, "\n", "")
 	result, errs = test.RunCmdWithError("prefop", result)
-	if len(errs) > 0 && !strings.Contains(errs, "not_found") {
+	if len(errs) > 0 && !strings.Contains(errs, "400 Bad Request") {
 		t.Fail()
 	}
 }

cmd_test/match_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/qiniu/qshell/v2/cmd_test/test"
+	"github.com/qiniu/qshell/v2/iqshell/common/utils"
 )
 
 func TestMatch(t *testing.T) {
@@ -100,7 +101,7 @@ func TestBatchMatch(t *testing.T) {
 	}
 
 	objectPath := filepath.Join(resultDir, test.Key)
-	_, _ = test.RunCmdWithError("get", test.Bucket, test.Key, "--domain", test.BucketDomain,
+	_, _ = test.RunCmdWithError("get", test.Bucket, test.Key, "--domain", utils.Endpoint(false, test.BucketDomain),
 		"-o", objectPath, "")
 	defer test.RemoveFile(objectPath)
 

cmd_test/test/config.go

@@ -66,7 +66,7 @@ http://domain/hello7_test.json
 
 	Key         = "hello1_test.json"
 	ImageKey    = "image.png"
-	KeyNotExist = "hello_mock.json"
+	KeyNotExist = "hello_mock_mock.json"
 	OriginKeys  = []string{"hello1.json", "hello2.json", "hello3.json", "hello4.json", "hello5.json", "hello6.json", "hello7.json"}
 	Keys        = []string{"hello1_test.json", "hello2_test.json", "hello3_test.json", "hello4_test.json", "hello5_test.json", "hello6_test.json", "hello7_test.json"}
 	KeysString  = `hello1_test.json

docs/get.md

@@ -25,7 +25,7 @@ $ qshell get --doc
 
 # 选项
 - -o/--outfile：保存在本地的文件路径；不指定，保存在当前文件夹，文件名使用存储空间中的名字【可选】
-- --domain：设置下载请求的域名，默认为空表示从存储源站下载，qshell 下载使用域名的优先级：1.配置选项中的下载请求域名(此参数) 2.bucket 绑定的域名(qshell 内部查询，无需配置) 3. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【可选】
+- --domain：指定下载请求的域名，当指定了下载域名则仅使用此下载域名进行下载；默认为空，此时 qshell 下载使用域名的优先级：1.bucket 绑定的域名(qshell 内部查询，无需配置) 2. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【可选】
 - --get-file-api: 当存储服务端支持 getfile 接口时才有效。【可选】
 - --public：空间是否为公开空间；为 `true` 时为公有空间，公有空间下载时不会对下载 URL 进行签名，可以提升 CDN 域名性能，默认为 `false`（私有空间）【可选】
 - --check-size: 下载后检测本地文件和服务端文件 size 的一致性。【可选】

docs/qdownload.md

@@ -69,7 +69,7 @@ $ qshell qdownload --doc
 - save_path_handler：指定一个回调函数；在构建文件的保存路径时，优先使用此选项进行构建，如果不配置则使用 $dest_dir + $文件分割符 + $Key 方式进行构建。文档下面有常用场景实例。此函数通过 Go 语言的模板实现，函数验证使用 func 命令，具体语法可参考 func 命令说明，handler 使用方式下方有示例可供参考 【可选】
 - check_size：下载后检测本地文件和服务端文件 size 的一致性，默认为 `false`。【可选】
 - check_hash：是否验证 hash，如果开启可能会耗费较长时间，默认为 `false` 【可选】
-- domain：设置下载请求的域名，默认为空表示从存储源站下载，qshell 下载使用域名的优先级：1.配置文件中的下载请求域名(此参数) 2.bucket 绑定的域名(qshell 内部查询，无需配置)  3. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【该功能默认需要计费，如果希望享受 10G 的免费流量，请自行配置此参数为 CDN 域名，如不配置，需支付源站流量费用，无法减免！！！】 【可选】
+- domain：指定下载请求的域名，当指定了下载域名则仅使用此下载域名进行下载；默认为空，此时 qshell 下载使用域名的优先级：1.bucket 绑定的域名(qshell 内部查询，无需配置) 2. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【该功能默认需要计费，如果希望享受 10G 的免费流量，请自行配置此参数为 CDN 域名，如不配置，需支付源站流量费用，无法减免！！！】 【可选】
 - referer：如果下载请求域名配置了域名白名单防盗链，需要指定一个允许访问的 referer 地址；默认为空 【可选】
 - public：空间是否为公开空间；为 `true` 时为公有空间，公有空间下载时不会对下载 URL 进行签名，可以提升 CDN 域名性能，默认为 `false`（私有空间）【可选】
 - enable_slice: 是否开启切片下载，需要注意 `slice_file_size_threshold` 切片阈值选项的配置，只有开启切片下载，并且下载的文件大小大于切片阈值方会启动切片下载。默认不开启。【可选】

iqshell/cdn/cdn.go

@@ -3,9 +3,10 @@ package cdn
 import (
 	"github.com/qiniu/go-sdk/v7/auth/qbox"
 	"github.com/qiniu/go-sdk/v7/cdn"
-	"github.com/qiniu/qshell/v2/iqshell/common/account"
+
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
 	"github.com/qiniu/qshell/v2/iqshell/common/log"
+	"github.com/qiniu/qshell/v2/iqshell/common/workspace"
 )
 
 const (
@@ -16,7 +17,7 @@ const (
 
 // GetCdnManager 获取CdnManager
 func getCdnManager() (cdnManager *cdn.CdnManager, err *data.CodeError) {
-	acc, gErr := account.GetAccount()
+	acc, gErr := workspace.GetAccount()
 	if gErr != nil {
 		err = data.NewEmptyError().AppendDescF("GetCdnManager error: %v\n", gErr)
 		return

iqshell/common/client/client.go

@@ -0,0 +1,32 @@
+package client
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/qiniu/go-sdk/v7/storage"
+)
+
+var defaultClient = storage.Client{
+	Client: &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   10 * time.Second,
+				KeepAlive: 10 * time.Second,
+			}).DialContext,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          4000,
+			MaxIdleConnsPerHost:   1000,
+			ResponseHeaderTimeout: 60 * time.Second,
+			IdleConnTimeout:       10 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	},
+}
+
+func DefaultStorageClient() storage.Client {
+	return defaultClient
+}