Skip to content

Commit 982ae0a

Browse files
author
jojoliang
committed
latest 0.7.23 stable
1 parent f42ee97 commit 982ae0a

File tree

9 files changed

+348
-48
lines changed

9 files changed

+348
-48
lines changed

‎ci.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"encoding/xml"
7+
"hash/crc64"
78
"io"
89
"net/http"
910
"os"
@@ -209,20 +210,24 @@ func (s *CIService) Put(ctx context.Context, name string, r io.Reader, opt *Obje
209210
if err := CheckReaderLen(r); err != nil {
210211
return nil, nil, err
211212
}
213+
totalBytes, err := GetReaderLen(r)
214+
if err != nil && opt != nil && opt.Listener != nil {
215+
return nil, nil, err
216+
}
217+
reader := TeeReader(r, nil, totalBytes, nil)
218+
if s.client.Conf.EnableCRC {
219+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
220+
}
212221
if opt != nil && opt.Listener != nil {
213-
totalBytes, err := GetReaderLen(r)
214-
if err != nil {
215-
return nil, nil, err
216-
}
217-
r = TeeReader(r, nil, totalBytes, opt.Listener)
222+
reader.listener = opt.Listener
218223
}
219224

220225
var res ImageProcessResult
221226
sendOpt := sendOptions{
222227
baseURL: s.client.BaseURL.BucketURL,
223228
uri: "/" + encodeURIComponent(name),
224229
method: http.MethodPut,
225-
body: r,
230+
body: reader,
226231
optHeader: opt,
227232
result: &res,
228233
}

‎cos.go‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222

2323
const (
2424
// Version current go sdk version
25-
Version = "0.7.22"
25+
Version = "0.7.23"
2626
userAgent = "cos-go-sdk-v5/" + Version
2727
contentTypeXML = "application/xml"
2828
defaultServiceBaseURL = "http://service.cos.myqcloud.com"
@@ -217,6 +217,18 @@ func (c *Client) doAPI(ctx context.Context, req *http.Request, result interface{
217217
return response, err
218218
}
219219

220+
// need CRC64 verification
221+
if reader, ok := req.Body.(*teeReader); ok {
222+
if c.Conf.EnableCRC && reader.writer != nil {
223+
localcrc := reader.Crc64()
224+
scoscrc := response.Header.Get("x-cos-hash-crc64ecma")
225+
icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
226+
if icoscrc != localcrc {
227+
return response, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
228+
}
229+
}
230+
}
231+
220232
if result != nil {
221233
if w, ok := result.(io.Writer); ok {
222234
io.Copy(w, resp.Body)

‎costesting/ci_test.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,37 @@ func (s *CosTestSuite) TestPutGetDeleteObjectByFile_10MB() {
476476
assert.Nil(s.T(), err, "remove local file Failed")
477477
}
478478

479+
func (s *CosTestSuite) TestPutGetDeleteObjectByUpload_10MB() {
480+
// Create tmp file
481+
filePath := "tmpfile" + time.Now().Format(time.RFC3339)
482+
newfile, err := os.Create(filePath)
483+
assert.Nil(s.T(), err, "create tmp file Failed")
484+
defer newfile.Close()
485+
486+
name := "test/objectUpload" + time.Now().Format(time.RFC3339)
487+
b := make([]byte, 1024*1024*10)
488+
_, err = rand.Read(b)
489+
490+
newfile.Write(b)
491+
opt := &cos.MultiUploadOptions{
492+
PartSize: 1,
493+
ThreadPoolSize: 3,
494+
}
495+
_, _, err = s.Client.Object.Upload(context.Background(), name, filePath, opt)
496+
assert.Nil(s.T(), err, "PutObject Failed")
497+
498+
// Over write tmp file
499+
_, err = s.Client.Object.GetToFile(context.Background(), name, filePath, nil)
500+
assert.Nil(s.T(), err, "HeadObject Failed")
501+
502+
_, err = s.Client.Object.Delete(context.Background(), name)
503+
assert.Nil(s.T(), err, "DeleteObject Failed")
504+
505+
// remove the local tmp file
506+
err = os.Remove(filePath)
507+
assert.Nil(s.T(), err, "remove local file Failed")
508+
}
509+
479510
func (s *CosTestSuite) TestPutGetDeleteObjectSpecialName() {
480511
f := strings.NewReader("test")
481512
name := s.SepFileName + time.Now().Format(time.RFC3339)

‎helper.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,8 @@ func GetReaderLen(reader io.Reader) (length int64, err error) {
149149
}
150150
case *io.LimitedReader:
151151
length = int64(v.N)
152+
case *LimitedReadCloser:
153+
length = int64(v.N)
152154
case FixedLengthReader:
153155
length = v.Size()
154156
default:
@@ -194,3 +196,29 @@ func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions
194196
}
195197
return optini
196198
}
199+
200+
// 浅拷贝ObjectPutOptions
201+
func cloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
202+
res := &ObjectPutOptions{
203+
&ACLHeaderOptions{},
204+
&ObjectPutHeaderOptions{},
205+
}
206+
if opt != nil {
207+
if opt.ACLHeaderOptions != nil {
208+
*res.ACLHeaderOptions = *opt.ACLHeaderOptions
209+
}
210+
if opt.ObjectPutHeaderOptions != nil {
211+
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
212+
}
213+
}
214+
return res
215+
}
216+
217+
// 浅拷贝ObjectUploadPartOptions
218+
func cloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions {
219+
var res ObjectUploadPartOptions
220+
if opt != nil {
221+
res = *opt
222+
}
223+
return &res
224+
}

‎object.go‎

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"encoding/xml"
77
"errors"
88
"fmt"
9+
"hash/crc64"
910
"io"
1011
"io/ioutil"
1112
"net/http"
@@ -147,7 +148,7 @@ type ObjectPutHeaderOptions struct {
147148
ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
148149
ContentType string `header:"Content-Type,omitempty" url:"-"`
149150
ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
150-
ContentLength int `header:"Content-Length,omitempty" url:"-"`
151+
ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
151152
ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
152153
Expect string `header:"Expect,omitempty" url:"-"`
153154
Expires string `header:"Expires,omitempty" url:"-"`
@@ -179,26 +180,27 @@ type ObjectPutOptions struct {
179180

180181
// Put Object请求可以将一个文件（Oject）上传至指定Bucket。
181182
//
182-
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength
183-
//
184183
// https://www.qcloud.com/document/product/436/7749
185184
func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
186185
if err := CheckReaderLen(r); err != nil {
187186
return nil, err
188187
}
188+
totalBytes, err := GetReaderLen(r)
189+
if err != nil && opt != nil && opt.Listener != nil {
190+
return nil, err
191+
}
192+
reader := TeeReader(r, nil, totalBytes, nil)
193+
if s.client.Conf.EnableCRC {
194+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
195+
}
189196
if opt != nil && opt.Listener != nil {
190-
totalBytes, err := GetReaderLen(r)
191-
if err != nil {
192-
return nil, err
193-
}
194-
r = TeeReader(r, nil, totalBytes, opt.Listener)
197+
reader.listener = opt.Listener
195198
}
196-
197199
sendOpt := sendOptions{
198200
baseURL: s.client.BaseURL.BucketURL,
199201
uri: "/" + encodeURIComponent(name),
200202
method: http.MethodPut,
201-
body: r,
203+
body: reader,
202204
optHeader: opt,
203205
}
204206
resp, err := s.client.send(ctx, &sendOpt)
@@ -556,38 +558,54 @@ type Results struct {
556558
err error
557559
}
558560

561+
func LimitReadCloser(r io.Reader, n int64) io.Reader {
562+
var lc LimitedReadCloser
563+
lc.R = r
564+
lc.N = n
565+
return &lc
566+
}
567+
568+
type LimitedReadCloser struct {
569+
io.LimitedReader
570+
}
571+
572+
func (lc *LimitedReadCloser) Close() error {
573+
if r, ok := lc.R.(io.ReadCloser); ok {
574+
return r.Close()
575+
}
576+
return nil
577+
}
578+
559579
func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
560580
for j := range jobs {
561-
fd, err := os.Open(j.FilePath)
562-
var res Results
563-
if err != nil {
564-
res.err = err
565-
res.PartNumber = j.Chunk.Number
566-
res.Resp = nil
567-
results <- &res
568-
}
569-
570-
// UploadPart do not support the chunk trsf, so need to add the content-length
571-
j.Opt.ContentLength = int(j.Chunk.Size)
581+
j.Opt.ContentLength = j.Chunk.Size
572582

573583
rt := j.RetryTimes
574584
for {
585+
// http.Request.Body can be Closed in request
586+
fd, err := os.Open(j.FilePath)
587+
var res Results
588+
if err != nil {
589+
res.err = err
590+
res.PartNumber = j.Chunk.Number
591+
res.Resp = nil
592+
results <- &res
593+
break
594+
}
575595
fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
576596
resp, err := s.UploadPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number,
577-
&io.LimitedReader{R: fd, N: j.Chunk.Size}, j.Opt)
597+
LimitReadCloser(fd, j.Chunk.Size), j.Opt)
578598
res.PartNumber = j.Chunk.Number
579599
res.Resp = resp
580600
res.err = err
581601
if err != nil {
582602
rt--
583603
if rt == 0 {
584-
fd.Close()
585604
results <- &res
586605
break
587606
}
588607
continue
589608
}
590-
fd.Close()
591609
results <- &res
592610
break
593611
}

‎object_part.go‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/xml"
66
"errors"
77
"fmt"
8+
"hash/crc64"
89
"io"
910
"net/http"
1011
"net/url"
@@ -47,7 +48,7 @@ func (s *ObjectService) InitiateMultipartUpload(ctx context.Context, name string
4748
type ObjectUploadPartOptions struct {
4849
Expect string `header:"Expect,omitempty" url:"-"`
4950
XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"`
50-
ContentLength int `header:"Content-Length,omitempty" url:"-"`
51+
ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
5152
ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
5253
XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
5354
XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
@@ -68,24 +69,36 @@ type ObjectUploadPartOptions struct {
6869
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ContentLength
6970
//
7071
// https://www.qcloud.com/document/product/436/7750
71-
func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *ObjectUploadPartOptions) (*Response, error) {
72+
func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, uopt *ObjectUploadPartOptions) (*Response, error) {
7273
if err := CheckReaderLen(r); err != nil {
7374
return nil, err
7475
}
75-
if opt != nil && opt.Listener != nil {
76-
totalBytes, err := GetReaderLen(r)
77-
if err != nil {
78-
return nil, err
76+
// opt 不为 nil
77+
opt := cloneObjectUploadPartOptions(uopt)
78+
totalBytes, err := GetReaderLen(r)
79+
if err != nil && opt.Listener != nil {
80+
return nil, err
81+
}
82+
// 分块上传不支持 Chunk 上传
83+
if err == nil {
84+
if opt != nil && opt.ContentLength == 0 {
85+
opt.ContentLength = totalBytes
7986
}
80-
r = TeeReader(r, nil, totalBytes, opt.Listener)
87+
}
88+
reader := TeeReader(r, nil, totalBytes, nil)
89+
if s.client.Conf.EnableCRC {
90+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
91+
}
92+
if opt != nil && opt.Listener != nil {
93+
reader.listener = opt.Listener
8194
}
8295
u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID)
8396
sendOpt := sendOptions{
8497
baseURL: s.client.BaseURL.BucketURL,
8598
uri: u,
8699
method: http.MethodPut,
87100
optHeader: opt,
88-
body: r,
101+
body: reader,
89102
}
90103
resp, err := s.client.send(ctx, &sendOpt)
91104
return resp, err

‎object_part_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ import (
55
"context"
66
"encoding/xml"
77
"fmt"
8+
"hash/crc64"
89
"io/ioutil"
910
"net/http"
1011
"reflect"
12+
"strconv"
1113
"testing"
1214
)
1315

@@ -102,11 +104,14 @@ func TestObjectService_UploadPart(t *testing.T) {
102104
testFormValues(t, r, vs)
103105

104106
b, _ := ioutil.ReadAll(r.Body)
107+
tb := crc64.MakeTable(crc64.ECMA)
108+
crc := crc64.Update(0, tb, b)
105109
v := string(b)
106110
want := "hello"
107111
if !reflect.DeepEqual(v, want) {
108112
t.Errorf("Object.UploadPart request body: %#v, want %#v", v, want)
109113
}
114+
w.Header().Add("x-cos-hash-crc64ecma", strconv.FormatUint(crc, 10))
110115
})
111116

112117
r := bytes.NewReader([]byte("hello"))

0 commit comments

Comments
 (0)