S3 bucket: Don't retry when ctx cancel (#5997)

* s3: no need to retry and wait when last error is context cancel or deadline exceeded

Signed-off-by: Ben Ye <[email protected]>

* add tests

Signed-off-by: Ben Ye <[email protected]>

* lint

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

pkg/storage/bucket/s3/bucket_client.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	"github.com/thanos-io/objstore"
 	"github.com/thanos-io/objstore/providers/s3"
@@ -127,6 +128,10 @@ func (b *BucketWithRetries) retry(ctx context.Context, f func() error, operation
 		if lastErr == nil {
 			return nil
 		}
+		// No need to retry when context was already canceled.
+		if errors.Is(lastErr, context.Canceled) || errors.Is(lastErr, context.DeadlineExceeded) {
+			return lastErr
+		}
 		if b.bucket.IsObjNotFoundErr(lastErr) || b.bucket.IsAccessDeniedErr(lastErr) {
 			return lastErr
 		}

pkg/storage/bucket/s3/bucket_client_test.go

@@ -34,6 +34,14 @@ func TestBucketWithRetries_ShouldRetry(t *testing.T) {
 			err:        errKeyDenied,
 			retryCount: 1,
 		},
+		"should not retry when context canceled": {
+			err:        context.Canceled,
+			retryCount: 1,
+		},
+		"should not retry when context deadline exceeded": {
+			err:        context.DeadlineExceeded,
+			retryCount: 1,
+		},
 	}
 
 	for name, tc := range cases {