Skip to content

Commit 146e400

Browse files
zhou-hongyuHongyu Zhou
zhou-hongyu
and
Hongyu Zhou
authored
[CONFIG-329][CONFIG-330] End to end snapshot integrity check (#124)
* refactor supervisor to use sdk v2 and add checksum * open another reader to get checksum * make generate * refactoring * hex encoding * don't calculate * use metadata * WIP test * wip * wip * WIP * WIP * WIP * fix install * WIP * wip * wip * chmod * install s3cmd from alpine * install aws-cli in alpine * install shasum in alpine * wip * test if sha value matches * wip * fix shell syntax error * add 5 attempts threshold * skip checksum validation if null * test unhappy path * revert false negative * address feedbacks * fix script * test unhappy path * revert false negative * using sha1 * update log to avoid confusion * fix wording --------- Co-authored-by: Hongyu Zhou <[email protected]>
1 parent 47950dd commit 146e400

File tree

7 files changed

+532
-131
lines changed

7 files changed

+532
-131
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers
1919
&& cp ${GOPATH}/bin/ctlstore-cli /usr/local/bin
2020

2121
FROM alpine
22-
RUN apk --no-cache add sqlite pigz
22+
RUN apk --no-cache add sqlite pigz aws-cli perl-utils jq
2323

2424
COPY --from=0 /go/src/github.com/segmentio/ctlstore/scripts/download.sh .
2525
COPY --from=0 /bin/chamber /bin/chamber

‎go.mod‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@ go 1.20
55
require (
66
github.com/AlekSi/pointer v1.0.0
77
github.com/aws/aws-sdk-go v1.37.8
8+
github.com/aws/aws-sdk-go-v2/config v1.18.40
9+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84
10+
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
811
github.com/fsnotify/fsnotify v1.5.1
912
github.com/go-sql-driver/mysql v1.4.1
10-
github.com/google/go-cmp v0.5.6
13+
github.com/google/go-cmp v0.5.8
1114
github.com/google/uuid v1.1.2
1215
github.com/gorilla/mux v1.7.3
1316
github.com/julienschmidt/httprouter v1.2.0
@@ -23,6 +26,22 @@ require (
2326
)
2427

2528
require (
29+
github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect
30+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
31+
github.com/aws/aws-sdk-go-v2/credentials v1.13.38 // indirect
32+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
33+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
34+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
35+
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
36+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
37+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
38+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
39+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
40+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
41+
github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 // indirect
42+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0 // indirect
43+
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 // indirect
44+
github.com/aws/smithy-go v1.14.2 // indirect
2645
github.com/davecgh/go-spew v1.1.1 // indirect
2746
github.com/jmespath/go-jmespath v0.4.0 // indirect
2847
github.com/mdlayher/genetlink v0.0.0-20190313224034-60417448a851 // indirect

‎go.sum‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,44 @@ github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJs
44
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
55
github.com/aws/aws-sdk-go v1.37.8 h1:9kywcbuz6vQuTf+FD+U7FshafrHzmqUCjgAEiLuIJ8U=
66
github.com/aws/aws-sdk-go v1.37.8/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
7+
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
8+
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
9+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
10+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
11+
github.com/aws/aws-sdk-go-v2/config v1.18.40 h1:dbu1llI/nTIL+r6sYHMeVLl99DM8J8/o1I4EPurnhLg=
12+
github.com/aws/aws-sdk-go-v2/config v1.18.40/go.mod h1:JjrCZQwSPGCoZRQzKHyZNNueaKO+kFaEy2sR6mCzd90=
13+
github.com/aws/aws-sdk-go-v2/credentials v1.13.38 h1:gDAuCdVlA4lmmgQhvpZlscwicloCqH44vkxLklGkQLA=
14+
github.com/aws/aws-sdk-go-v2/credentials v1.13.38/go.mod h1:sD4G/Ybgp6s89mWIES3Xn97CsRLpxvz9uVSdv0UxY8I=
15+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg=
16+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8=
17+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84 h1:LENrVcqnWTyI8fbIUCvxAMe+fXbREIaXzcR8WPwco1U=
18+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84/go.mod h1:LHxCiYAStsgps4srke7HujyADd504MSkNXjLpOtICTc=
19+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
20+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
21+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
22+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
23+
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0=
24+
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo=
25+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw=
26+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
27+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0=
28+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
29+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk=
30+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
31+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
32+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
33+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg=
34+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
35+
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk=
36+
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
37+
github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 h1:AR/hlTsCyk1CwlyKnPFvIMvnONydRjDDRT9OGb0i+/g=
38+
github.com/aws/aws-sdk-go-v2/service/sso v1.14.0/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4=
39+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0 h1:vbgiXuhtn49+erlPrgIvQ+J32rg1HseaPf8lEpKbkxQ=
40+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4=
41+
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk=
42+
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU=
43+
github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
44+
github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
745
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
846
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
947
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -24,8 +62,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
2462
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
2563
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
2664
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
27-
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
28-
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
65+
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
66+
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
2967
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
3068
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
3169
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
@@ -155,7 +193,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
155193
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
156194
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
157195
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
158-
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
159196
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
160197
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
161198
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

‎pkg/supervisor/archived_snapshot.go‎

Lines changed: 68 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,18 @@ package supervisor
33
import (
44
"bufio"
55
"context"
6+
"crypto/sha1"
7+
"encoding/base64"
8+
"fmt"
69
"io"
710
"net/url"
811
"os"
912
"strings"
1013
"time"
1114

12-
"github.com/aws/aws-sdk-go/aws/session"
13-
"github.com/aws/aws-sdk-go/service/s3/s3manager"
15+
"github.com/aws/aws-sdk-go-v2/config"
16+
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
17+
"github.com/aws/aws-sdk-go-v2/service/s3"
1418
"github.com/pkg/errors"
1519
"github.com/segmentio/events/v2"
1620
"github.com/segmentio/stats/v4"
@@ -52,7 +56,7 @@ type s3Snapshot struct {
5256
Bucket string
5357
Key string
5458
sendToS3Func sendToS3Func
55-
s3Uploader S3Uploader
59+
s3Client S3Client
5660
}
5761

5862
func (c *s3Snapshot) Upload(ctx context.Context, path string) error {
@@ -71,6 +75,12 @@ func (c *s3Snapshot) Upload(ctx context.Context, path string) error {
7175
key = key[1:]
7276
}
7377
var reader io.Reader = bufio.NewReaderSize(f, 1024*32) // use a 32K buffer for reading
78+
79+
cs, err := getChecksum(path)
80+
if err != nil {
81+
return errors.Wrap(err, "generate file Checksum")
82+
}
83+
7484
var gpr *gzipCompressionReader
7585
if strings.HasSuffix(key, ".gz") {
7686
events.Log("Compressing s3 payload with GZIP")
@@ -80,7 +90,7 @@ func (c *s3Snapshot) Upload(ctx context.Context, path string) error {
8090
events.Log("Uploading %{file}s (%d bytes) to %{bucket}s/%{key}s", path, size, c.Bucket, key)
8191

8292
start := time.Now()
83-
if err = c.sendToS3(ctx, key, c.Bucket, reader); err != nil {
93+
if err = c.sendToS3(ctx, key, c.Bucket, reader, cs); err != nil {
8494
return errors.Wrap(err, "send to s3")
8595
}
8696
stats.Observe("ldb-upload-time", time.Since(start), stats.T("compressed", isCompressed(gpr)))
@@ -97,42 +107,83 @@ func (c *s3Snapshot) Upload(ctx context.Context, path string) error {
97107
return nil
98108
}
99109

110+
func getChecksum(path string) (string, error) {
111+
f, err := os.OpenFile(path, os.O_RDONLY, 0)
112+
if err != nil {
113+
return "", errors.Wrap(err, "opening file")
114+
}
115+
defer f.Close()
116+
117+
h := sha1.New()
118+
if _, err := io.Copy(h, f); err != nil {
119+
events.Log("failed to generate sha1 of snapshot", err)
120+
}
121+
122+
cs := base64.StdEncoding.EncodeToString(h.Sum(nil))
123+
events.Log("base64 encoding of sha1: %s", cs)
124+
125+
return cs, nil
126+
}
127+
100128
func isCompressed(gpr *gzipCompressionReader) string {
101129
if gpr == nil {
102130
return "false"
103131
}
104132
return "true"
105133
}
106134

107-
func (c *s3Snapshot) sendToS3(ctx context.Context, key string, bucket string, body io.Reader) error {
135+
type BucketBasics struct {
136+
S3Client S3Client
137+
}
138+
139+
func (c *s3Snapshot) sendToS3(ctx context.Context, key string, bucket string, body io.Reader, cs string) error {
108140
if c.sendToS3Func != nil {
109141
return c.sendToS3Func(ctx, key, bucket, body)
110142
}
111-
ul, err := c.getS3Uploader()
143+
144+
client, err := c.getS3Client()
112145
if err != nil {
113146
return err
114147
}
115-
output, err := ul.UploadWithContext(ctx, &s3manager.UploadInput{
116-
Bucket: &bucket,
117-
Key: &key,
118-
Body: body,
148+
149+
var basics = BucketBasics{
150+
S3Client: client,
151+
}
152+
var partMiBs int64 = 16
153+
uploader := manager.NewUploader(basics.S3Client, func(u *manager.Uploader) {
154+
u.PartSize = partMiBs * 1024 * 1024
155+
})
156+
157+
output, err := uploader.Upload(ctx, &s3.PutObjectInput{
158+
Bucket: &bucket,
159+
Key: &key,
160+
Body: body,
161+
ChecksumAlgorithm: "sha256",
162+
Metadata: map[string]string{
163+
"checksum": cs,
164+
},
119165
})
120166
if err == nil {
121167
events.Log("Wrote to S3 location: %s", output.Location)
168+
} else {
169+
events.Log("Couldn't upload s3 snapshot to %v:%v. Here's why: %v\n",
170+
bucket, key, err)
122171
}
123172
return errors.Wrap(err, "upload with context")
124173
}
125174

126-
func (c *s3Snapshot) getS3Uploader() (S3Uploader, error) {
127-
if c.s3Uploader != nil {
128-
return c.s3Uploader, nil
175+
func (c *s3Snapshot) getS3Client() (S3Client, error) {
176+
if c.s3Client != nil {
177+
return c.s3Client, nil
129178
}
130-
sess, err := session.NewSession()
179+
cfg, err := config.LoadDefaultConfig(context.Background())
180+
131181
if err != nil {
132-
return nil, errors.Wrap(err, "creating aws session")
182+
panic(fmt.Sprintf("failed loading config, %v", err))
133183
}
134-
uploader := s3manager.NewUploader(sess)
135-
return uploader, nil
184+
185+
client := s3.NewFromConfig(cfg)
186+
return client, nil
136187
}
137188

138189
func archivedSnapshotFromURL(URL string) (archivedSnapshot, error) {

0 commit comments

Comments
 (0)