Merge pull request #116 from agin719/cos-v4-dev

Cos v4 dev

Author: agin719

cos.go

@@ -22,7 +22,7 @@ import (
 
 const (
 	// Version current go sdk version
-	Version               = "0.7.23"
+	Version               = "0.7.24"
 	userAgent             = "cos-go-sdk-v5/" + Version
 	contentTypeXML        = "application/xml"
 	defaultServiceBaseURL = "http://service.cos.myqcloud.com"
@@ -71,7 +71,8 @@ func NewBucketURL(bucketName, region string, secure bool) *url.URL {
 }
 
 type Config struct {
-	EnableCRC bool
+	EnableCRC        bool
+	RequestBodyClose bool
 }
 
 // Client is a client manages communication with the COS API.
@@ -119,7 +120,8 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client {
 		UserAgent: userAgent,
 		BaseURL:   baseURL,
 		Conf: &Config{
-			EnableCRC: true,
+			EnableCRC:        true,
+			RequestBodyClose: false,
 		},
 	}
 	c.common.client = c
@@ -182,6 +184,9 @@ func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method s
 	if c.Host != "" {
 		req.Host = c.Host
 	}
+	if c.Conf.RequestBodyClose {
+		req.Close = true
+	}
 	return
 }
 

helper.go

@@ -16,6 +16,7 @@ import (
 
 // 单次上传文件最大为5GB
 const singleUploadMaxLength = 5 * 1024 * 1024 * 1024
+const singleUploadThreshold = 32 * 1024 * 1024
 
 // 计算 md5 或 sha1 时的分块大小
 const calDigestBlockSize = 1024 * 1024 * 10

object.go

@@ -217,14 +217,24 @@ func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt
 
 // PutFromFile put object from local file
 // Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory
-func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*Response, error) {
-	fd, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
+func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) {
+	nr := 0
+	for nr < 3 {
+		fd, e := os.Open(filePath)
+		if e != nil {
+			err = e
+			return
+		}
+		resp, err = s.Put(ctx, name, fd, opt)
+		if err != nil {
+			nr++
+			fd.Close()
+			continue
+		}
+		fd.Close()
+		break
 	}
-	defer fd.Close()
-
-	return s.Put(ctx, name, fd, opt)
+	return
 }
 
 // ObjectCopyHeaderOptions is the head option of the Copy
@@ -778,9 +788,12 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string
 		}
 		defer fd.Close()
 		localcrc, err = calCRC64(fd)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 	// filesize=0 , use simple upload
-	if partNum == 0 {
+	if partNum == 0 || partNum == 1 {
 		var opt0 *ObjectPutOptions
 		if opt.OptIni != nil {
 			opt0 = &ObjectPutOptions{

object_test.go

@@ -375,13 +375,13 @@ func TestObjectService_Upload(t *testing.T) {
 	}
 	defer os.Remove(filePath)
 	// 源文件内容
-	b := make([]byte, 1024*1024*10)
+	b := make([]byte, 1024*1024*33)
 	_, err = rand.Read(b)
 	newfile.Write(b)
 	newfile.Close()
 
 	// 已上传内容, 10个分块
-	rb := make([][]byte, 10)
+	rb := make([][]byte, 33)
 	uploadid := "test-cos-multiupload-uploadid"
 	partmap := make(map[int64]int)
 	mux.HandleFunc("/test.go.upload", func(w http.ResponseWriter, r *http.Request) {
@@ -423,9 +423,7 @@ func TestObjectService_Upload(t *testing.T) {
 				// 完成分块上传
 				tb := crc64.MakeTable(crc64.ECMA)
 				crc := uint64(0)
-				ccv := make([]uint64, 10)
-				for i, v := range rb {
-					ccv[i] = crc64.Update(0, crc64.MakeTable(crc64.ECMA), v)
+				for _, v := range rb {
 					crc = crc64.Update(crc, tb, v)
 				}
 				w.Header().Add("x-cos-hash-crc64ecma", strconv.FormatUint(crc, 10))
@@ -450,3 +448,69 @@ func TestObjectService_Upload(t *testing.T) {
 		t.Fatalf("Object.Upload returned error: %v", err)
 	}
 }
+
+func TestObjectService_Upload2(t *testing.T) {
+	setup()
+	defer teardown()
+
+	filePath := "tmpfile" + time.Now().Format(time.RFC3339)
+	newfile, err := os.Create(filePath)
+	if err != nil {
+		t.Fatalf("create tmp file failed")
+	}
+	defer os.Remove(filePath)
+	// 源文件内容
+	b := make([]byte, 1024*1024*3)
+	_, err = rand.Read(b)
+	newfile.Write(b)
+	newfile.Close()
+
+	tb := crc64.MakeTable(crc64.ECMA)
+	realcrc := crc64.Update(0, tb, b)
+	name := "test/hello.txt"
+	retry := 0
+	final := 4
+	mux.HandleFunc("/test/hello.txt", func(w http.ResponseWriter, r *http.Request) {
+		testMethod(t, r, http.MethodPut)
+		testHeader(t, r, "x-cos-acl", "private")
+		testHeader(t, r, "Content-Type", "text/html")
+
+		if retry%2 == 0 {
+			bs, _ := ioutil.ReadAll(r.Body)
+			crc := crc64.Update(0, tb, bs)
+			if !reflect.DeepEqual(bs, b) {
+				t.Errorf("Object.Put request body Error")
+			}
+			if !reflect.DeepEqual(crc, realcrc) {
+				t.Errorf("Object.Put crc: %v, want: %v", crc, realcrc)
+			}
+			w.Header().Add("x-cos-hash-crc64ecma", strconv.FormatUint(crc, 10))
+			if retry != final {
+				w.WriteHeader(http.StatusGatewayTimeout)
+			}
+		} else {
+			w.Header().Add("x-cos-hash-crc64ecma", "123456789")
+		}
+	})
+
+	mopt := &MultiUploadOptions{
+		OptIni: &InitiateMultipartUploadOptions{
+			ObjectPutHeaderOptions: &ObjectPutHeaderOptions{
+				ContentType: "text/html",
+			},
+			ACLHeaderOptions: &ACLHeaderOptions{
+				XCosACL: "private",
+			},
+		},
+	}
+	for retry <= final {
+		_, _, err := client.Object.Upload(context.Background(), name, filePath, mopt)
+		if retry < final && err == nil {
+			t.Fatalf("Error must not nil when retry < final")
+		}
+		if retry == final && err != nil {
+			t.Fatalf("Put Error: %v", err)
+		}
+		retry++
+	}
+}