Skip to content

Commit 4284b43

Browse files
agin719agin719
authored
Merge pull request #227 from tencentyun/feature_jojoliang_07c6039e
Feature jojoliang 07c6039e
2 parents 5c79b3d + b110e20 commit 4284b43

File tree

8 files changed

+388
-46
lines changed

8 files changed

+388
-46
lines changed

‎ci_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,7 @@ func TestCIService_Put(t *testing.T) {
612612
&ObjectPutHeaderOptions{
613613
XOptionHeader: &http.Header{},
614614
},
615+
nil,
615616
}
616617
opt.XOptionHeader.Add("Pic-Operations", EncodePicOperations(pic))
617618
res, _, err := client.CI.Put(context.Background(), name, f, opt)
@@ -734,6 +735,7 @@ func TestCIService_PutFromFile(t *testing.T) {
734735
&ObjectPutHeaderOptions{
735736
XOptionHeader: &http.Header{},
736737
},
738+
nil,
737739
}
738740
opt.XOptionHeader.Add("Pic-Operations", EncodePicOperations(pic))
739741
res, _, err := client.CI.PutFromFile(context.Background(), name, filePath, opt)

‎cos.go‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@ var (
4343
hostPrefix = regexp.MustCompile(`^(http://|https://){0,1}([a-z0-9-]+-[0-9]+\.){0,1}((cos|cos-internal|cos-website|ci)\.[a-z-1]+|file)\.(myqcloud\.com|tencentcos\.cn).*$`)
4444
invalidBucketErr = fmt.Errorf("invalid bucket format, please check your cos.BaseURL")
4545

46-
switchHost = regexp.MustCompile(`([a-z0-9-]+-[0-9]+\.)((cos|cos-website)\.[a-z-1]+)\.(myqcloud\.com)(:[0-9]+){0,1}$`)
47-
oldDomainSuffix = ".myqcloud.com"
48-
newDomainSuffix = ".tencentcos.cn"
46+
switchHost = regexp.MustCompile(`([a-z0-9-]+-[0-9]+\.)((cos|cos-website)\.[a-z-1]+)\.(myqcloud\.com)(:[0-9]+){0,1}$`)
47+
accelerateDomainSuffix = "accelerate.myqcloud.com"
48+
oldDomainSuffix = ".myqcloud.com"
49+
newDomainSuffix = ".tencentcos.cn"
4950
)
5051

5152
// BaseURL 访问各 API 所需的基础 URL
@@ -365,6 +366,10 @@ func toSwitchHost(oldURL *url.URL) *url.URL {
365366
newURL, _ := url.Parse(oldURL.String())
366367
hostAndPort := strings.SplitN(newURL.Host, ":", 2)
367368
newHost := hostAndPort[0]
369+
// 加速域名不切换
370+
if strings.HasSuffix(newHost, accelerateDomainSuffix) {
371+
return oldURL
372+
}
368373
newHost = newHost[:len(newHost)-len(oldDomainSuffix)] + newDomainSuffix
369374
if len(hostAndPort) > 1 {
370375
newHost += ":" + hostAndPort[1]
@@ -373,6 +378,24 @@ func toSwitchHost(oldURL *url.URL) *url.URL {
373378
return newURL
374379
}
375380

381+
func (c *Client) CheckRetrieable(u *url.URL, resp *Response, err error) (*url.URL, bool) {
382+
res := u
383+
if err != nil && err != invalidBucketErr {
384+
// 不重试
385+
if resp != nil && resp.StatusCode < 500 {
386+
return res, false
387+
}
388+
if c.Conf.RetryOpt.AutoSwitchHost {
389+
// 收不到报文 或者 不存在RequestId
390+
if resp == nil || resp.Header.Get("X-Cos-Request-Id") == "" {
391+
res = toSwitchHost(u)
392+
}
393+
}
394+
return res, true
395+
}
396+
return res, false
397+
}
398+
376399
func (c *Client) doRetry(ctx context.Context, opt *sendOptions) (resp *Response, err error) {
377400
if opt.body != nil {
378401
if _, ok := opt.body.(io.Reader); ok {
@@ -384,29 +407,21 @@ func (c *Client) doRetry(ctx context.Context, opt *sendOptions) (resp *Response,
384407
if c.Conf.RetryOpt.Count > 0 {
385408
count = c.Conf.RetryOpt.Count
386409
}
387-
interval := c.Conf.RetryOpt.Interval
410+
var retrieable bool
388411
for nr := 0; nr < count; nr++ {
389412
resp, err = c.send(ctx, opt)
390-
if err != nil && err != invalidBucketErr {
391-
// 不重试
392-
if resp != nil && resp.StatusCode < 500 {
393-
break
394-
}
395-
if c.Conf.RetryOpt.AutoSwitchHost {
396-
// 收不到报文 或者 不存在RequestId
397-
if resp == nil || resp.Header.Get("X-Cos-Request-Id") == "" {
398-
opt.baseURL = toSwitchHost(opt.baseURL)
399-
}
400-
}
401-
if interval > 0 && nr+1 < count {
402-
time.Sleep(interval)
413+
opt.baseURL, retrieable = c.CheckRetrieable(opt.baseURL, resp, err)
414+
if retrieable {
415+
if c.Conf.RetryOpt.Interval > 0 && nr+1 < count {
416+
time.Sleep(c.Conf.RetryOpt.Interval)
403417
}
404418
continue
405419
}
406420
break
407421
}
408422
return
409423
}
424+
410425
func (c *Client) send(ctx context.Context, opt *sendOptions) (resp *Response, err error) {
411426
req, err := c.newRequest(ctx, opt.baseURL, opt.uri, opt.method, opt.body, opt.optQuery, opt.optHeader)
412427
if err != nil {

‎cos_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,4 +222,12 @@ func Test_SwitchHost(t *testing.T) {
222222
if res.String() != want {
223223
t.Errorf("toSwitchHost failed, expect: %v, res: %v", want, res.String())
224224
}
225+
226+
u, _ = url.Parse("https://example-125000000.cos.accelerate.myqcloud.com:443/123")
227+
res = toSwitchHost(u)
228+
want = "https://example-125000000.cos.accelerate.myqcloud.com:443/123"
229+
if res.String() != want {
230+
t.Errorf("toSwitchHost failed, expect: %v, res: %v", want, res.String())
231+
}
232+
225233
}

‎helper.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
233233
res := &ObjectPutOptions{
234234
&ACLHeaderOptions{},
235235
&ObjectPutHeaderOptions{},
236+
nil,
236237
}
237238
if opt != nil {
238239
if opt.ACLHeaderOptions != nil {
@@ -243,6 +244,9 @@ func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
243244
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
244245
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
245246
}
247+
if opt.innerSwitchURL != nil {
248+
res.innerSwitchURL = opt.innerSwitchURL
249+
}
246250
}
247251
return res
248252
}

‎object.go‎

Lines changed: 61 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,9 @@ type ObjectPutHeaderOptions struct {
332332
type ObjectPutOptions struct {
333333
*ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
334334
*ObjectPutHeaderOptions `header:",omitempty" url:"-" xml:"-"`
335+
336+
// PutFromFile 使用
337+
innerSwitchURL *url.URL `header:"-" url:"-" xml:"-"`
335338
}
336339

337340
// Put Object请求可以将一个文件（Oject）上传至指定Bucket。
@@ -358,27 +361,56 @@ func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt
358361
opt.ContentLength = totalBytes
359362
}
360363
}
361-
reader := TeeReader(r, nil, totalBytes, nil)
362-
if s.client.Conf.EnableCRC {
363-
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
364-
}
365-
if opt != nil && opt.Listener != nil {
366-
reader.listener = opt.Listener
367-
}
368-
sendOpt := sendOptions{
369-
baseURL: s.client.BaseURL.BucketURL,
370-
uri: "/" + encodeURIComponent(name),
371-
method: http.MethodPut,
372-
body: reader,
373-
optHeader: opt,
364+
// 如果是io.Seeker，则重试
365+
count := 1
366+
var position int64
367+
if seeker, ok := r.(io.Seeker); ok {
368+
// 记录原始位置
369+
position, err = seeker.Seek(0, io.SeekCurrent)
370+
if err == nil && s.client.Conf.RetryOpt.Count > 0 {
371+
count = s.client.Conf.RetryOpt.Count
372+
}
373+
}
374+
var resp *Response
375+
var retrieable bool
376+
sUrl := s.client.BaseURL.BucketURL
377+
if opt.innerSwitchURL != nil {
378+
sUrl = opt.innerSwitchURL
379+
}
380+
for nr := 0; nr < count; nr++ {
381+
reader := TeeReader(r, nil, totalBytes, nil)
382+
if s.client.Conf.EnableCRC {
383+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
384+
}
385+
if opt != nil && opt.Listener != nil {
386+
reader.listener = opt.Listener
387+
}
388+
sendOpt := sendOptions{
389+
baseURL: sUrl,
390+
uri: "/" + encodeURIComponent(name),
391+
method: http.MethodPut,
392+
body: reader,
393+
optHeader: opt,
394+
}
395+
resp, err = s.client.send(ctx, &sendOpt)
396+
sUrl, retrieable = s.client.CheckRetrieable(sUrl, resp, err)
397+
if retrieable && nr+1 < count {
398+
if seeker, ok := r.(io.Seeker); ok {
399+
_, e := seeker.Seek(position, io.SeekStart)
400+
if e != nil {
401+
break
402+
}
403+
continue
404+
}
405+
}
406+
break
374407
}
375-
resp, err := s.client.send(ctx, &sendOpt)
376-
377408
return resp, err
378409
}
379410

380411
// PutFromFile put object from local file
381-
func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) {
412+
func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, uopt *ObjectPutOptions) (resp *Response, err error) {
413+
opt := CloneObjectPutOptions(uopt)
382414
nr := 0
383415
for nr < 3 {
384416
fd, e := os.Open(filePath)
@@ -390,6 +422,12 @@ func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath s
390422
if err != nil {
391423
nr++
392424
fd.Close()
425+
if s.client.Conf.RetryOpt.AutoSwitchHost {
426+
// 收不到报文 或者 不存在RequestId
427+
if resp == nil || resp.Header.Get("X-Cos-Request-Id") == "" {
428+
opt.innerSwitchURL = toSwitchHost(s.client.BaseURL.BucketURL)
429+
}
430+
}
393431
continue
394432
}
395433
fd.Close()
@@ -900,6 +938,12 @@ func worker(ctx context.Context, s *ObjectService, jobs <-chan *Jobs, results ch
900938
results <- &res
901939
break
902940
}
941+
if s.client.Conf.RetryOpt.AutoSwitchHost {
942+
// 收不到报文 或者 不存在RequestId
943+
if resp == nil || resp.Header.Get("X-Cos-Request-Id") == "" {
944+
j.Opt.innerSwitchURL = toSwitchHost(s.client.BaseURL.BucketURL)
945+
}
946+
}
903947
time.Sleep(time.Millisecond)
904948
continue
905949
}
@@ -1128,6 +1172,7 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string
11281172
opt0 = &ObjectPutOptions{
11291173
opt.OptIni.ACLHeaderOptions,
11301174
opt.OptIni.ObjectPutHeaderOptions,
1175+
nil,
11311176
}
11321177
}
11331178
rsp, err := s.PutFromFile(ctx, name, filepath, opt0)

‎object_part.go‎

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ type ObjectUploadPartOptions struct {
6060
XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
6161
// 上传进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil
6262
Listener ProgressListener `header:"-" url:"-" xml:"-"`
63+
64+
// Upload方法使用
65+
innerSwitchURL *url.URL `header:"-" url:"-" xml:"-"`
6366
}
6467

6568
// UploadPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。
@@ -93,22 +96,51 @@ func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, p
9396
opt.ContentLength = totalBytes
9497
}
9598
}
96-
reader := TeeReader(r, nil, totalBytes, nil)
97-
if s.client.Conf.EnableCRC {
98-
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
99+
// 如果是io.Seeker，则重试
100+
count := 1
101+
var position int64
102+
if seeker, ok := r.(io.Seeker); ok {
103+
// 记录原始位置
104+
position, err = seeker.Seek(0, io.SeekCurrent)
105+
if err == nil && s.client.Conf.RetryOpt.Count > 0 {
106+
count = s.client.Conf.RetryOpt.Count
107+
}
99108
}
100-
if opt != nil && opt.Listener != nil {
101-
reader.listener = opt.Listener
109+
var resp *Response
110+
var retrieable bool
111+
sUrl := s.client.BaseURL.BucketURL
112+
if opt.innerSwitchURL != nil {
113+
sUrl = opt.innerSwitchURL
102114
}
103-
u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID)
104-
sendOpt := sendOptions{
105-
baseURL: s.client.BaseURL.BucketURL,
106-
uri: u,
107-
method: http.MethodPut,
108-
optHeader: opt,
109-
body: reader,
115+
for nr := 0; nr < count; nr++ {
116+
reader := TeeReader(r, nil, totalBytes, nil)
117+
if s.client.Conf.EnableCRC {
118+
reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
119+
}
120+
if opt != nil && opt.Listener != nil {
121+
reader.listener = opt.Listener
122+
}
123+
u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID)
124+
sendOpt := sendOptions{
125+
baseURL: sUrl,
126+
uri: u,
127+
method: http.MethodPut,
128+
optHeader: opt,
129+
body: reader,
130+
}
131+
resp, err = s.client.send(ctx, &sendOpt)
132+
sUrl, retrieable = s.client.CheckRetrieable(sUrl, resp, err)
133+
if retrieable && nr+1 < count {
134+
if seeker, ok := r.(io.Seeker); ok {
135+
_, e := seeker.Seek(position, io.SeekStart)
136+
if e != nil {
137+
break
138+
}
139+
continue
140+
}
141+
}
142+
break
110143
}
111-
resp, err := s.client.send(ctx, &sendOpt)
112144
return resp, err
113145
}
114146

0 commit comments

Comments
 (0)