Merge pull request #398 from YangSen-qn/nfs

support sequential read file

Author: bachue

cmd/upload.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"github.com/qiniu/go-sdk/v7/storage"
+	"github.com/spf13/cobra"
+
 	"github.com/qiniu/qshell/v2/docs"
 	"github.com/qiniu/qshell/v2/iqshell"
 	"github.com/qiniu/qshell/v2/iqshell/common/config"
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
 	"github.com/qiniu/qshell/v2/iqshell/storage/object/upload/operations"
-	"github.com/spf13/cobra"
 )
 
 var uploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
@@ -71,6 +72,7 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVarP(&info.OverwriteExportFilePath, "overwrite-list", "w", "", "upload success (overwrite) file list")
 	cmd.Flags().IntVar(&info.Info.WorkerCount, "thread-count", 1, "multiple thread count")
 	cmd.Flags().IntVar(&info.UploadConfig.WorkerCount, "worker-count", 3, "the number of concurrently uploaded parts of a single file in resumable upload")
+	cmd.Flags().BoolVar(&info.UploadConfig.SequentialReadFile, "sequential-read-file", false, "File reading is sequential and does not involve skipping; when enabled, the uploading fragment data will be loaded into the memory. This option may increase file upload speed for mounted network filesystems.")
 
 	cmd.Flags().BoolVarP(&info.ResumableAPIV2, "resumable-api-v2", "", false, "use resumable upload v2 APIs to upload")
 	cmd.Flags().Int64Var(&info.ResumableAPIV2PartSize, "resumable-api-v2-part-size", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload")
@@ -203,6 +205,7 @@ var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVarP(&info.MimeType, "mimetype", "t", "", "file mime type")
 	cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "", false, "overwrite the file of same key in bucket")
 	cmd.Flags().BoolVarP(&info.UseResumeV2, "resumable-api-v2", "", false, "use resumable upload v2 APIs to upload")
+	cmd.Flags().BoolVar(&info.SequentialReadFile, "sequential-read-file", false, "File reading is sequential and does not involve skipping; when enabled, the uploading fragment data will be loaded into the memory. This option may increase file upload speed for mounted network filesystems.")
 
 	cmd.Flags().Int64VarP(&info.ChunkSize, "resumable-api-v2-part-size", "", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload, default 4M")
 	cmd.Flags().Int64VarP(&info.ChunkSize, "v2-part-size", "", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload, same to --resumable-api-v2-part-size")

cmd_test/aa_pre_test.go

@@ -4,9 +4,10 @@ package cmd
 
 import (
 	"fmt"
+	"testing"
+
 	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
-	"testing"
 )
 
 func TestCmd(t *testing.T) {

cmd_test/user.go

@@ -3,10 +3,11 @@
 package cmd
 
 import (
-	"github.com/qiniu/qshell/v2/cmd"
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"os"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd"
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 var accessKey = test.AccessKey

docs/qupload.md

@@ -84,6 +84,7 @@ $ qshell qupload --doc
 - resumable_api_v2：使用分片 V2 进行上传，默认为 `false` 使用分片 V1 。【可选】
 - resumable_api_v2_part_size：使用分片 V2 进行上传时定制分片大小，默认 4194304（4M） 。【可选】
 - put_threshold：上传阈值，上传文件大小超过此值会使用分片上传，不超过使用表单上传；单位：B，默认为 8388608（8M） 。【可选】
+- sequential_read_file: 文件读为顺序读，不涉及跳读；开启后，上传中的分片数据会被加载至内存。此选项可能会增加挂载网络文件系统的文件上传速度。默认是：false。 【可选】
 - record_root：上传记录信息保存路径，包括日志文件和上传进度文件；默认为 `qshell` 上传目录；【可选】
   - 通过 `-L` 指定工作目录时，`record_root` 则为此工作目录/qdownload/$jobId， 
   - 未通过 `-L` 指定工作目录时为 `用户目录/.qshell/users/$CurrentUserName/qdownload/$jobId`

docs/qupload2.md

@@ -42,6 +42,7 @@ Flags:
       --rescan-local                     rescan local dir to upload newly add files
       --resumable-api-v2                 use resumable upload v2 APIs to upload
       --resumable-api-v2-part-size int   the part size when use resumable upload v2 APIs to upload (default 4194304)
+      --sequential-read-file             File reading is sequential and does not involve skipping; when enabled, the uploading fragment data will be loaded into the memory. This option may increase file upload speed for mounted network filesystems.
       --skip-file-prefixes string        skip files with these file prefixes
       --skip-fixed-strings string        skip files with the fixed string in the name
       --skip-path-prefixes string        skip files with these relative path prefixes

docs/rput.md

@@ -38,6 +38,7 @@ $ qshell rput --doc
 - -T/--callbackHost：上传回调HOST, 必须和 CallbackUrls 一起指定。 【可选】
 - --resumable-api-v2：使用分片上传 API V2 进行上传，默认为 `false`, 使用 V1 上传。【可选】
 - --resumable-api-v2-part-size：使用分片上传 API V2 进行上传时的分片大小，默认为 4M 。【可选】
+- --sequential-read-file: 文件读为顺序读，不涉及跳读；开启后，上传中的分片数据会被加载至内存。此选项可能会增加挂载网络文件系统的文件上传速度。默认是：false。【可选】
 
 # 示例
 1 上传本地文件 `/Users/jemy/Documents/qiniu.mp4` 到空间 `if-pbl` 里面。

iqshell/common/account/actions.go

@@ -7,11 +7,12 @@ import (
 	"strings"
 
 	"github.com/qiniu/go-sdk/v7/auth/qbox"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
 	"github.com/qiniu/qshell/v2/iqshell/common/config"
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
 	"github.com/qiniu/qshell/v2/iqshell/common/log"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
 // 保存账户信息到账户文件中
@@ -96,9 +97,9 @@ func SaveToDB(acc Account, accountOver bool) (err *data.CodeError) {
 }
 
 func getAccount(pt string) (account Account, err *data.CodeError) {
-	accountFh, openErr := os.Open(pt)
+	accountFh, openErr := os.OpenFile(info.AccountPath, os.O_RDWR, 0600)
 	if openErr != nil {
-		err = data.NewEmptyError().AppendDescF("Open account file error, %s, please use `account` to set Id and SecretKey first", openErr)
+		err = data.NewEmptyError().AppendDescF("Get account error, %s, please use `account` to set Id and SecretKey first", openErr)
 		return
 	}
 	defer accountFh.Close()
@@ -115,12 +116,48 @@ func getAccount(pt string) (account Account, err *data.CodeError) {
 	}
 
 	acc, dErr := decrypt(string(accountBytes))
-	if dErr != nil {
-		err = data.NewEmptyError().AppendDescF("Decrypt account bytes: %v", dErr)
+	if dErr == nil {
+		account = acc
 		return
 	}
-	account = acc
-	return
+
+	oldError := data.NewEmptyError().AppendDescF("Decrypt account bytes: %s, you can delete account file(%s) and use `account` command to reset the account", dErr, pt)
+	if len(acc.Name) == 0 {
+		return account, oldError
+	}
+
+	accounts, lErr := LookUp(acc.Name)
+	if lErr != nil || len(accounts) == 0 {
+		return account, oldError
+	}
+
+	account = accounts[0]
+
+	// 尝试替换错误缓存，失败也没关系
+	jsonStr, mErr := account.value()
+	if mErr != nil {
+		log.WarningF("get account, get json error:%s", mErr)
+		return account, nil
+	}
+
+	_, sErr := accountFh.Seek(0, io.SeekStart)
+	if sErr != nil {
+		log.WarningF("get account, file seek error:%s", sErr)
+		return account, nil
+	}
+
+	tErr := accountFh.Truncate(0)
+	if tErr != nil {
+		log.WarningF("get account, file truncate error:%s", tErr)
+		return account, nil
+	}
+
+	_, wErr := accountFh.WriteString(jsonStr)
+	if wErr != nil {
+		log.WarningF("get account, file write error:%s", wErr)
+	}
+
+	return account, nil
 }
 
 // qshell 会记录当前的user信息，当切换账户后， 老的账户信息会记录下来

iqshell/common/account/crypt.go

@@ -2,9 +2,10 @@ package account
 
 import (
 	"encoding/base64"
-	"github.com/qiniu/qshell/v2/iqshell/common/data"
 	"strings"
 
+	"github.com/qiniu/qshell/v2/iqshell/common/data"
+
 	"github.com/qiniu/qshell/v2/iqshell/common/utils"
 )
 
@@ -15,6 +16,15 @@ func splits(joinStr string) []string {
 // 对保存在account.json中的文件字符串进行揭秘操作, 返回Account
 func decrypt(joinStr string) (acc Account, err *data.CodeError) {
 	ss := splits(joinStr)
+	if len(ss) > 0 {
+		acc.Name = ss[0]
+	}
+
+	if len(ss) != 3 {
+		err = data.NewEmptyError().AppendDescF("account json style format error")
+		return
+	}
+
 	name, accessKey, encryptedKey := ss[0], ss[1], ss[2]
 	if name == "" || accessKey == "" || encryptedKey == "" {
 		err = data.NewEmptyError().AppendDescF("name, accessKey and encryptedKey should not be empty")

iqshell/common/utils/crypto.go

@@ -6,6 +6,7 @@ import (
 	"crypto/cipher"
 	"crypto/md5"
 	"encoding/hex"
+
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
 )
 
@@ -42,6 +43,9 @@ func AesDecrypt(crypted, key []byte) ([]byte, *data.CodeError) {
 
 	blockMode.CryptBlocks(origData, crypted)
 	origData = PKCS5UnPadding(origData)
+	if len(origData) == 0 {
+		return nil, data.NewEmptyError().AppendDesc("data format error")
+	}
 	return origData, nil
 }
 
@@ -56,5 +60,9 @@ func PKCS5Padding(ciphertext []byte, blockSize int) []byte {
 func PKCS5UnPadding(origData []byte) []byte {
 	length := len(origData)
 	unpadding := int(origData[length-1])
+	if unpadding >= length {
+		return nil
+	}
+
 	return origData[:(length - unpadding)]
 }

iqshell/common/workspace/load.go

@@ -1,10 +1,12 @@
 package workspace
 
 import (
+	"path/filepath"
+
 	"github.com/qiniu/go-sdk/v7/auth"
 	"github.com/qiniu/go-sdk/v7/storage"
+
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
-	"path/filepath"
 
 	"github.com/qiniu/qshell/v2/iqshell/common/account"
 	"github.com/qiniu/qshell/v2/iqshell/common/config"
@@ -151,6 +153,7 @@ func loadUserInfo() {
 		}
 	} else {
 		userDir = filepath.Join(workspaceDir, usersDirName, defaultUserDirName)
+		log.WarningF("load user error:%s", err)
 	}
 
 	log.DebugF("user dir:%s", userDir)