Skip to content

Commit e279629

Browse files
authored
Merge pull request #52 from buildkite/keithduncan/retry-s3-503
Retry S3 and S3 Manager requests that receive a 503 response
2 parents 04df5f3 + 60bbda3 commit e279629

File tree

4 files changed

+104
-64
lines changed

4 files changed

+104
-64
lines changed

s3secrets-helper/go.mod

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,11 @@ module github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v
22

33
go 1.15
44

5-
require github.com/aws/aws-sdk-go v1.35.14
5+
require (
6+
github.com/aws/aws-sdk-go-v2 v1.9.2 // indirect
7+
github.com/aws/aws-sdk-go-v2/config v1.8.3
8+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 // indirect
9+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4
10+
github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1
11+
github.com/aws/smithy-go v1.8.0
12+
)

s3secrets-helper/go.sum

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,44 @@
1-
github.com/aws/aws-sdk-go v1.35.14 h1:nucVVXXjAr9UkmYCBWxQWRuYa5KOlaXjuJGg2ulW0K0=
2-
github.com/aws/aws-sdk-go v1.35.14/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
3-
github.com/buildkite/elastic-ci-stack-s3-secrets-hooks v1.3.0 h1:CRdPqLjYECJY0cgs24E4ZwUJ2qlmcavN7W1jo+5ujnQ=
1+
github.com/aws/aws-sdk-go-v2 v1.9.2 h1:dUFQcMNZMLON4BOe273pl0filK9RqyQMhCK/6xssL6s=
2+
github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
3+
github.com/aws/aws-sdk-go-v2/config v1.8.3 h1:o5583X4qUfuRrOGOgmOcDgvr5gJVSu57NK08cWAhIDk=
4+
github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw=
5+
github.com/aws/aws-sdk-go-v2/credentials v1.4.3 h1:LTdD5QhK073MpElh9umLLP97wxphkgVC/OjQaEbBwZA=
6+
github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ=
7+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 h1:9tfxW/icbSu98C2pcNynm5jmDwU3/741F11688B6QnU=
8+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM=
9+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 h1:TnU1cY51027j/MQeFy7DIgk1UuzJY+wLFYqXceY/fiE=
10+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4/go.mod h1:Ex7XQmbFmgFHrjUX6TN3mApKW5Hglyga+F7wZHTtYhA=
11+
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4 h1:leSJ6vCqtPpTmBIgE7044B1wql1E4n//McF+mEgNrYg=
12+
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ=
13+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8=
14+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU=
15+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2 h1:r7jel2aa4d9Duys7wEmWqDd5ebpC9w6Kxu6wIjjp18E=
16+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8=
17+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.2 h1:RnZjLgtCGLsF2xYYksy0yrx6xPvKG9BYv29VfK4p/J8=
18+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.2/go.mod h1:np7TMuJNT83O0oDOSF8i4dF3dvGqA6hPYYo6YYkzgRA=
19+
github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1 h1:z+P3r4LrwdudLKBoEVWxIORrk4sVg4/iqpG3+CS53AY=
20+
github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1/go.mod h1:CQe/KvWV1AqRc65KqeJjrLzr5X2ijnFTTVzJW0VBRCI=
21+
github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 h1:pZwkxZbspdqRGzddDB92bkZBoB7lg85sMRE7OqdB3V0=
22+
github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk=
23+
github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 h1:ol2Y5DWqnJeKqNd8th7JWzBtqu63xpOfs1Is+n1t8/4=
24+
github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g=
25+
github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc=
26+
github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
27+
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
428
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
29+
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
30+
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
31+
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
532
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
633
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
34+
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
735
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
8-
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
36+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
937
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1038
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
11-
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
12-
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
13-
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
14-
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
39+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
40+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
41+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
1542
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
43+
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
1644
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

s3secrets-helper/s3/s3.go

Lines changed: 58 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,73 @@
11
package s3
22

33
import (
4+
"context"
5+
"errors"
6+
"fmt"
47
"log"
58
"io/ioutil"
69
"os"
710

8-
"github.com/aws/aws-sdk-go/aws"
9-
"github.com/aws/aws-sdk-go/aws/awserr"
10-
"github.com/aws/aws-sdk-go/aws/ec2metadata"
11-
"github.com/aws/aws-sdk-go/aws/session"
12-
"github.com/aws/aws-sdk-go/service/s3"
13-
"github.com/aws/aws-sdk-go/service/s3/s3manager"
11+
"github.com/aws/aws-sdk-go-v2/config"
12+
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
13+
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
14+
"github.com/aws/aws-sdk-go-v2/service/s3"
15+
"github.com/aws/aws-sdk-go-v2/service/s3/types"
16+
"github.com/aws/smithy-go"
1417
"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
1518
)
1619

17-
const envDefaultRegion = "AWS_DEFAULT_REGION"
18-
1920
type Client struct {
20-
s3 *s3.S3
21+
s3 *s3.Client
2122
bucket string
2223
}
2324

24-
func New(log *log.Logger, bucket string) (*Client, error) {
25-
sess, err := session.NewSession()
26-
if err != nil {
27-
return nil, err
25+
func getRegion(ctx context.Context) (string, error) {
26+
if region := os.Getenv("AWS_DEFAULT_REGION"); len(region) > 0 {
27+
return region, nil
2828
}
2929

30-
currentRegion := os.Getenv(envDefaultRegion)
31-
// Discover our executing region using the IMDS
32-
if currentRegion == "" {
33-
idms := ec2metadata.New(sess)
34-
currentRegion, _ = idms.Region()
35-
}
36-
// Fall back to us-east-1 :(
37-
if currentRegion == "" {
38-
currentRegion = "us-east-1"
30+
imdsClient := imds.New(imds.Options{})
31+
if result, err := imdsClient.GetRegion(ctx, nil); err == nil {
32+
if len(result.Region) > 0 {
33+
return result.Region, nil
34+
}
3935
}
4036

41-
log.Printf("Discovered current region as %q\n", currentRegion)
37+
return "", errors.New("Unknown current region")
38+
}
39+
40+
func New(log *log.Logger, bucket string) (*Client, error) {
41+
ctx := context.Background()
4242

4343
// Using the current region (or a guess) find where the bucket lives
44-
bucketRegion, err := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, currentRegion)
44+
45+
region, err := getRegion(ctx)
4546
if err != nil {
46-
return nil, err
47+
// Ignore error and fallback to us-east-1 for bucket lookup
48+
region = "us-east-1"
4749
}
4850

49-
log.Printf("Discovered bucket region as %q\n", bucketRegion)
51+
config, err := config.LoadDefaultConfig(ctx,
52+
config.WithRegion(region),
53+
)
54+
if err != nil {
55+
return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
56+
}
5057

51-
sess, err = session.NewSession(&aws.Config{
52-
Region: &bucketRegion,
53-
})
58+
log.Printf("Discovered current region as %q\n", config.Region)
59+
60+
bucketRegion, err := manager.GetBucketRegion(ctx, s3.NewFromConfig(config), bucket)
5461
if err != nil {
55-
return nil, err
62+
return nil, fmt.Errorf("Could not discover the region for bucket %q: (%v)", bucket, err)
5663
}
64+
65+
log.Printf("Discovered bucket region as %q\n", bucketRegion)
66+
67+
config.Region = bucketRegion
68+
5769
return &Client{
58-
s3: s3.New(sess),
70+
s3: s3.NewFromConfig(config),
5971
bucket: bucket,
6072
}, nil
6173
}
@@ -69,23 +81,26 @@ func (c *Client) Bucket() (string) {
6981
// sentinel.ErrNotFound and sentinel.ErrForbidden are returned for those cases.
7082
// Other errors are returned verbatim.
7183
func (c *Client) Get(key string) ([]byte, error) {
72-
out, err := c.s3.GetObject(&s3.GetObjectInput{
84+
out, err := c.s3.GetObject(context.TODO(), &s3.GetObjectInput{
7385
Bucket: &c.bucket,
7486
Key: &key,
7587
})
7688
if err != nil {
77-
if aerr, ok := err.(awserr.Error); ok {
78-
switch aerr.Code() {
79-
case "NoSuchKey":
80-
return nil, sentinel.ErrNotFound
81-
case "Forbidden":
89+
var noSuchKey *types.NoSuchKey
90+
if errors.As(err, &noSuchKey) {
91+
return nil, sentinel.ErrNotFound
92+
}
93+
94+
// Possible values can be found at https://docs.aws.amazon.com/AmazonS3/latest/API/API_Error.html
95+
var apiErr smithy.APIError
96+
if errors.As(err, &apiErr) {
97+
code := apiErr.ErrorCode()
98+
if code == "AccessDenied" {
8299
return nil, sentinel.ErrForbidden
83-
default:
84-
return nil, aerr
85100
}
86-
} else {
87-
return nil, err
88101
}
102+
103+
return nil, fmt.Errorf("Could not GetObject (%s) in bucket (%s). Ensure your IAM Identity has s3:GetObject permission for this key and bucket. (%v)", key, c.bucket, err)
89104
}
90105
defer out.Body.Close()
91106
// we probably should return io.Reader or io.ReadCloser rather than []byte,
@@ -98,18 +113,8 @@ func (c *Client) Get(key string) ([]byte, error) {
98113
// 404 Not Found and 403 Forbidden return false without error.
99114
// Other errors result in false with an error.
100115
func (c *Client) BucketExists() (bool, error) {
101-
if _, err := c.s3.HeadBucket(&s3.HeadBucketInput{Bucket: &c.bucket}); err != nil {
102-
if aerr, ok := err.(awserr.Error); ok {
103-
switch aerr.Code() {
104-
// https://github.com/aws/aws-sdk-go/issues/2593#issuecomment-491436818
105-
case "NoSuchBucket", "NotFound":
106-
return false, nil
107-
default: // e.g. NoCredentialProviders, Forbidden
108-
return false, aerr
109-
}
110-
} else {
111-
return false, err
112-
}
116+
if _, err := c.s3.HeadBucket(context.TODO(), &s3.HeadBucketInput{Bucket: &c.bucket}); err != nil {
117+
return false, fmt.Errorf("Could not HeadBucket (%s). Ensure your IAM Identity has s3:ListBucket permission for this bucket. (%v)", c.bucket, err)
113118
}
114119
return true, nil
115120
}

s3secrets-helper/sentinel/errors.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import "errors"
66

77
var (
88
// ErrNotFound indicates something was not found
9-
ErrNotFound = errors.New("Forbidden")
9+
ErrNotFound = errors.New("NotFound")
1010

1111
// ErrForbidden indicates something was forbidden
12-
ErrForbidden = errors.New("NotFound")
12+
ErrForbidden = errors.New("Forbidden")
1313
)

0 commit comments

Comments
 (0)