Skip to content

Commit 95ac8f1

Browse files
agin719agin719
authored
Merge pull request #113 from agin719/cos-v4-dev
latest 0.7.23 stable
2 parents f42ee97 + 17c220d commit 95ac8f1

File tree

9 files changed

+379
-50
lines changed

9 files changed

+379
-50
lines changed

‎ci.go‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"encoding/xml"
7+
"hash/crc64"
78
"io"
89
"net/http"
910
"os"
@@ -205,24 +206,35 @@ func (s *CIService) GetVideoAuditingJob(ctx context.Context, jobid string) (*Get
205206
}
206207

207208
// ci put https://cloud.tencent.com/document/product/460/18147
208-
func (s *CIService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*ImageProcessResult, *Response, error) {
209+
func (s *CIService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*ImageProcessResult, *Response, error) {
209210
if err := CheckReaderLen(r); err != nil {
210211
return nil, nil, err
211212
}
212-
if opt != nil && opt.Listener != nil {
213-
totalBytes, err := GetReaderLen(r)
214-
if err != nil {
215-
return nil, nil, err
213+
opt := cloneObjectPutOptions(uopt)
214+
totalBytes, err := GetReaderLen(r)
215+
if err != nil && opt != nil && opt.Listener != nil {
216+
return nil, nil, err
217+
}
218+
if err == nil {
219+
// 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传
220+
if opt != nil && opt.ContentLength == 0 && IsLenReader(r) {
221+
opt.ContentLength = totalBytes
216222
}
217-
r = TeeReader(r, nil, totalBytes, opt.Listener)
223+
}
224+
reader := TeeReader(r, nil, totalBytes, nil)
225+
if s.client.Conf.EnableCRC {
226+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
227+
}
228+
if opt != nil && opt.Listener != nil {
229+
reader.listener = opt.Listener
218230
}
219231

220232
var res ImageProcessResult
221233
sendOpt := sendOptions{
222234
baseURL: s.client.BaseURL.BucketURL,
223235
uri: "/" + encodeURIComponent(name),
224236
method: http.MethodPut,
225-
body: r,
237+
body: reader,
226238
optHeader: opt,
227239
result: &res,
228240
}

‎cos.go‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222

2323
const (
2424
// Version current go sdk version
25-
Version = "0.7.22"
25+
Version = "0.7.23"
2626
userAgent = "cos-go-sdk-v5/" + Version
2727
contentTypeXML = "application/xml"
2828
defaultServiceBaseURL = "http://service.cos.myqcloud.com"
@@ -217,6 +217,18 @@ func (c *Client) doAPI(ctx context.Context, req *http.Request, result interface{
217217
return response, err
218218
}
219219

220+
// need CRC64 verification
221+
if reader, ok := req.Body.(*teeReader); ok {
222+
if c.Conf.EnableCRC && reader.writer != nil {
223+
localcrc := reader.Crc64()
224+
scoscrc := response.Header.Get("x-cos-hash-crc64ecma")
225+
icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
226+
if icoscrc != localcrc {
227+
return response, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
228+
}
229+
}
230+
}
231+
220232
if result != nil {
221233
if w, ok := result.(io.Writer); ok {
222234
io.Copy(w, resp.Body)

‎costesting/ci_test.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,37 @@ func (s *CosTestSuite) TestPutGetDeleteObjectByFile_10MB() {
476476
assert.Nil(s.T(), err, "remove local file Failed")
477477
}
478478

479+
func (s *CosTestSuite) TestPutGetDeleteObjectByUpload_10MB() {
480+
// Create tmp file
481+
filePath := "tmpfile" + time.Now().Format(time.RFC3339)
482+
newfile, err := os.Create(filePath)
483+
assert.Nil(s.T(), err, "create tmp file Failed")
484+
defer newfile.Close()
485+
486+
name := "test/objectUpload" + time.Now().Format(time.RFC3339)
487+
b := make([]byte, 1024*1024*10)
488+
_, err = rand.Read(b)
489+
490+
newfile.Write(b)
491+
opt := &cos.MultiUploadOptions{
492+
PartSize: 1,
493+
ThreadPoolSize: 3,
494+
}
495+
_, _, err = s.Client.Object.Upload(context.Background(), name, filePath, opt)
496+
assert.Nil(s.T(), err, "PutObject Failed")
497+
498+
// Over write tmp file
499+
_, err = s.Client.Object.GetToFile(context.Background(), name, filePath, nil)
500+
assert.Nil(s.T(), err, "HeadObject Failed")
501+
502+
_, err = s.Client.Object.Delete(context.Background(), name)
503+
assert.Nil(s.T(), err, "DeleteObject Failed")
504+
505+
// remove the local tmp file
506+
err = os.Remove(filePath)
507+
assert.Nil(s.T(), err, "remove local file Failed")
508+
}
509+
479510
func (s *CosTestSuite) TestPutGetDeleteObjectSpecialName() {
480511
f := strings.NewReader("test")
481512
name := s.SepFileName + time.Now().Format(time.RFC3339)

‎helper.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,8 @@ func GetReaderLen(reader io.Reader) (length int64, err error) {
149149
}
150150
case *io.LimitedReader:
151151
length = int64(v.N)
152+
case *LimitedReadCloser:
153+
length = int64(v.N)
152154
case FixedLengthReader:
153155
length = v.Size()
154156
default:
@@ -157,6 +159,20 @@ func GetReaderLen(reader io.Reader) (length int64, err error) {
157159
return
158160
}
159161

162+
func IsLenReader(reader io.Reader) bool {
163+
switch reader.(type) {
164+
case *bytes.Buffer:
165+
return true
166+
case *bytes.Reader:
167+
return true
168+
case *strings.Reader:
169+
return true
170+
default:
171+
return false
172+
}
173+
return false
174+
}
175+
160176
func CheckReaderLen(reader io.Reader) error {
161177
nlen, err := GetReaderLen(reader)
162178
if err != nil || nlen < singleUploadMaxLength {
@@ -194,3 +210,29 @@ func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions
194210
}
195211
return optini
196212
}
213+
214+
// 浅拷贝ObjectPutOptions
215+
func cloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
216+
res := &ObjectPutOptions{
217+
&ACLHeaderOptions{},
218+
&ObjectPutHeaderOptions{},
219+
}
220+
if opt != nil {
221+
if opt.ACLHeaderOptions != nil {
222+
*res.ACLHeaderOptions = *opt.ACLHeaderOptions
223+
}
224+
if opt.ObjectPutHeaderOptions != nil {
225+
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
226+
}
227+
}
228+
return res
229+
}
230+
231+
// 浅拷贝ObjectUploadPartOptions
232+
func cloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions {
233+
var res ObjectUploadPartOptions
234+
if opt != nil {
235+
res = *opt
236+
}
237+
return &res
238+
}

‎object.go‎

Lines changed: 50 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"encoding/xml"
77
"errors"
88
"fmt"
9+
"hash/crc64"
910
"io"
1011
"io/ioutil"
1112
"net/http"
@@ -147,7 +148,7 @@ type ObjectPutHeaderOptions struct {
147148
ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
148149
ContentType string `header:"Content-Type,omitempty" url:"-"`
149150
ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
150-
ContentLength int `header:"Content-Length,omitempty" url:"-"`
151+
ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
151152
ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
152153
Expect string `header:"Expect,omitempty" url:"-"`
153154
Expires string `header:"Expires,omitempty" url:"-"`
@@ -179,26 +180,34 @@ type ObjectPutOptions struct {
179180

180181
// Put Object请求可以将一个文件（Oject）上传至指定Bucket。
181182
//
182-
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength
183-
//
184183
// https://www.qcloud.com/document/product/436/7749
185-
func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
184+
func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*Response, error) {
186185
if err := CheckReaderLen(r); err != nil {
187186
return nil, err
188187
}
189-
if opt != nil && opt.Listener != nil {
190-
totalBytes, err := GetReaderLen(r)
191-
if err != nil {
192-
return nil, err
188+
opt := cloneObjectPutOptions(uopt)
189+
totalBytes, err := GetReaderLen(r)
190+
if err != nil && opt != nil && opt.Listener != nil {
191+
return nil, err
192+
}
193+
if err == nil {
194+
// 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传
195+
if opt != nil && opt.ContentLength == 0 && IsLenReader(r) {
196+
opt.ContentLength = totalBytes
193197
}
194-
r = TeeReader(r, nil, totalBytes, opt.Listener)
195198
}
196-
199+
reader := TeeReader(r, nil, totalBytes, nil)
200+
if s.client.Conf.EnableCRC {
201+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
202+
}
203+
if opt != nil && opt.Listener != nil {
204+
reader.listener = opt.Listener
205+
}
197206
sendOpt := sendOptions{
198207
baseURL: s.client.BaseURL.BucketURL,
199208
uri: "/" + encodeURIComponent(name),
200209
method: http.MethodPut,
201-
body: r,
210+
body: reader,
202211
optHeader: opt,
203212
}
204213
resp, err := s.client.send(ctx, &sendOpt)
@@ -556,38 +565,54 @@ type Results struct {
556565
err error
557566
}
558567

568+
func LimitReadCloser(r io.Reader, n int64) io.Reader {
569+
var lc LimitedReadCloser
570+
lc.R = r
571+
lc.N = n
572+
return &lc
573+
}
574+
575+
type LimitedReadCloser struct {
576+
io.LimitedReader
577+
}
578+
579+
func (lc *LimitedReadCloser) Close() error {
580+
if r, ok := lc.R.(io.ReadCloser); ok {
581+
return r.Close()
582+
}
583+
return nil
584+
}
585+
559586
func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
560587
for j := range jobs {
561-
fd, err := os.Open(j.FilePath)
562-
var res Results
563-
if err != nil {
564-
res.err = err
565-
res.PartNumber = j.Chunk.Number
566-
res.Resp = nil
567-
results <- &res
568-
}
569-
570-
// UploadPart do not support the chunk trsf, so need to add the content-length
571-
j.Opt.ContentLength = int(j.Chunk.Size)
588+
j.Opt.ContentLength = j.Chunk.Size
572589

573590
rt := j.RetryTimes
574591
for {
592+
// http.Request.Body can be Closed in request
593+
fd, err := os.Open(j.FilePath)
594+
var res Results
595+
if err != nil {
596+
res.err = err
597+
res.PartNumber = j.Chunk.Number
598+
res.Resp = nil
599+
results <- &res
600+
break
601+
}
575602
fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
576603
resp, err := s.UploadPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number,
577-
&io.LimitedReader{R: fd, N: j.Chunk.Size}, j.Opt)
604+
LimitReadCloser(fd, j.Chunk.Size), j.Opt)
578605
res.PartNumber = j.Chunk.Number
579606
res.Resp = resp
580607
res.err = err
581608
if err != nil {
582609
rt--
583610
if rt == 0 {
584-
fd.Close()
585611
results <- &res
586612
break
587613
}
588614
continue
589615
}
590-
fd.Close()
591616
results <- &res
592617
break
593618
}

‎object_part.go‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/xml"
66
"errors"
77
"fmt"
8+
"hash/crc64"
89
"io"
910
"net/http"
1011
"net/url"
@@ -47,7 +48,7 @@ func (s *ObjectService) InitiateMultipartUpload(ctx context.Context, name string
4748
type ObjectUploadPartOptions struct {
4849
Expect string `header:"Expect,omitempty" url:"-"`
4950
XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"`
50-
ContentLength int `header:"Content-Length,omitempty" url:"-"`
51+
ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
5152
ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
5253
XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
5354
XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
@@ -68,24 +69,37 @@ type ObjectUploadPartOptions struct {
6869
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ContentLength
6970
//
7071
// https://www.qcloud.com/document/product/436/7750
71-
func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *ObjectUploadPartOptions) (*Response, error) {
72+
func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, uopt *ObjectUploadPartOptions) (*Response, error) {
7273
if err := CheckReaderLen(r); err != nil {
7374
return nil, err
7475
}
75-
if opt != nil && opt.Listener != nil {
76-
totalBytes, err := GetReaderLen(r)
77-
if err != nil {
78-
return nil, err
76+
// opt 不为 nil
77+
opt := cloneObjectUploadPartOptions(uopt)
78+
totalBytes, err := GetReaderLen(r)
79+
if err != nil && opt.Listener != nil {
80+
return nil, err
81+
}
82+
// 分块上传不支持 Chunk 上传
83+
if err == nil {
84+
// 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader需用户指定ContentLength
85+
if opt != nil && opt.ContentLength == 0 && IsLenReader(r) {
86+
opt.ContentLength = totalBytes
7987
}
80-
r = TeeReader(r, nil, totalBytes, opt.Listener)
88+
}
89+
reader := TeeReader(r, nil, totalBytes, nil)
90+
if s.client.Conf.EnableCRC {
91+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
92+
}
93+
if opt != nil && opt.Listener != nil {
94+
reader.listener = opt.Listener
8195
}
8296
u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID)
8397
sendOpt := sendOptions{
8498
baseURL: s.client.BaseURL.BucketURL,
8599
uri: u,
86100
method: http.MethodPut,
87101
optHeader: opt,
88-
body: r,
102+
body: reader,
89103
}
90104
resp, err := s.client.send(ctx, &sendOpt)
91105
return resp, err

‎object_part_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ import (
55
"context"
66
"encoding/xml"
77
"fmt"
8+
"hash/crc64"
89
"io/ioutil"
910
"net/http"
1011
"reflect"
12+
"strconv"
1113
"testing"
1214
)
1315

@@ -102,11 +104,14 @@ func TestObjectService_UploadPart(t *testing.T) {
102104
testFormValues(t, r, vs)
103105

104106
b, _ := ioutil.ReadAll(r.Body)
107+
tb := crc64.MakeTable(crc64.ECMA)
108+
crc := crc64.Update(0, tb, b)
105109
v := string(b)
106110
want := "hello"
107111
if !reflect.DeepEqual(v, want) {
108112
t.Errorf("Object.UploadPart request body: %#v, want %#v", v, want)
109113
}
114+
w.Header().Add("x-cos-hash-crc64ecma", strconv.FormatUint(crc, 10))
110115
})
111116

112117
r := bytes.NewReader([]byte("hello"))

0 commit comments

Comments
 (0)