diff --git a/Dockerfile b/Dockerfile index 72e9c3d8..a70bf5d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers && cp ${GOPATH}/bin/ctlstore-cli /usr/local/bin FROM alpine -RUN apk --no-cache add sqlite pigz +RUN apk --no-cache add sqlite pigz aws-cli perl-utils jq COPY --from=0 /go/src/github.com/segmentio/ctlstore/scripts/download.sh . COPY --from=0 /bin/chamber /bin/chamber diff --git a/go.mod b/go.mod index b6204e36..0e1cad6a 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,12 @@ go 1.20 require ( github.com/AlekSi/pointer v1.0.0 github.com/aws/aws-sdk-go v1.37.8 + github.com/aws/aws-sdk-go-v2/config v1.18.40 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/fsnotify/fsnotify v1.5.1 github.com/go-sql-driver/mysql v1.4.1 - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.8 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.3 github.com/julienschmidt/httprouter v1.2.0 @@ -23,6 +26,22 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.38 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/mdlayher/genetlink v0.0.0-20190313224034-60417448a851 // indirect diff --git a/go.sum b/go.sum index 578f101a..64a25a05 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,44 @@ github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJs github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/aws/aws-sdk-go v1.37.8 h1:9kywcbuz6vQuTf+FD+U7FshafrHzmqUCjgAEiLuIJ8U= github.com/aws/aws-sdk-go v1.37.8/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2/config v1.18.40 h1:dbu1llI/nTIL+r6sYHMeVLl99DM8J8/o1I4EPurnhLg= +github.com/aws/aws-sdk-go-v2/config v1.18.40/go.mod h1:JjrCZQwSPGCoZRQzKHyZNNueaKO+kFaEy2sR6mCzd90= +github.com/aws/aws-sdk-go-v2/credentials v1.13.38 h1:gDAuCdVlA4lmmgQhvpZlscwicloCqH44vkxLklGkQLA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.38/go.mod h1:sD4G/Ybgp6s89mWIES3Xn97CsRLpxvz9uVSdv0UxY8I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84 h1:LENrVcqnWTyI8fbIUCvxAMe+fXbREIaXzcR8WPwco1U= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84/go.mod h1:LHxCiYAStsgps4srke7HujyADd504MSkNXjLpOtICTc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 h1:AR/hlTsCyk1CwlyKnPFvIMvnONydRjDDRT9OGb0i+/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.14.0/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0 h1:vbgiXuhtn49+erlPrgIvQ+J32rg1HseaPf8lEpKbkxQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk= +github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -24,8 +62,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= @@ -155,7 +193,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/pkg/supervisor/archived_snapshot.go b/pkg/supervisor/archived_snapshot.go index 8b2b3bc9..bd5ace73 100644 --- a/pkg/supervisor/archived_snapshot.go +++ b/pkg/supervisor/archived_snapshot.go @@ -3,14 +3,18 @@ package supervisor import ( "bufio" "context" + "crypto/sha1" + "encoding/base64" + "fmt" "io" "net/url" "os" "strings" "time" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/pkg/errors" "github.com/segmentio/events/v2" "github.com/segmentio/stats/v4" @@ -52,7 +56,7 @@ type s3Snapshot struct { Bucket string Key string sendToS3Func sendToS3Func - s3Uploader S3Uploader + s3Client S3Client } func (c *s3Snapshot) Upload(ctx context.Context, path string) error { @@ -71,6 +75,12 @@ func (c *s3Snapshot) Upload(ctx context.Context, path string) error { key = key[1:] } var reader io.Reader = bufio.NewReaderSize(f, 1024*32) // use a 32K buffer for reading + + cs, err := getChecksum(path) + if err != nil { + return errors.Wrap(err, "generate file Checksum") + } + var gpr *gzipCompressionReader if strings.HasSuffix(key, ".gz") { events.Log("Compressing s3 payload with GZIP") @@ -80,7 +90,7 @@ func (c *s3Snapshot) Upload(ctx context.Context, path string) error { events.Log("Uploading %{file}s (%d bytes) to %{bucket}s/%{key}s", path, size, c.Bucket, key) start := time.Now() - if err = c.sendToS3(ctx, key, c.Bucket, reader); err != nil { + if err = c.sendToS3(ctx, key, c.Bucket, reader, cs); err != nil { return errors.Wrap(err, "send to s3") } stats.Observe("ldb-upload-time", time.Since(start), stats.T("compressed", isCompressed(gpr))) @@ -97,6 +107,24 @@ func (c *s3Snapshot) Upload(ctx context.Context, path string) error { return nil } +func getChecksum(path string) (string, error) { + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return "", errors.Wrap(err, "opening file") + } + defer f.Close() + + h := sha1.New() + if _, err := io.Copy(h, f); err != nil { + events.Log("failed to generate sha1 of snapshot", err) + } + + cs := base64.StdEncoding.EncodeToString(h.Sum(nil)) + events.Log("base64 encoding of sha1: %s", cs) + + return cs, nil +} + func isCompressed(gpr *gzipCompressionReader) string { if gpr == nil { return "false" @@ -104,35 +132,58 @@ func isCompressed(gpr *gzipCompressionReader) string { return "true" } -func (c *s3Snapshot) sendToS3(ctx context.Context, key string, bucket string, body io.Reader) error { +type BucketBasics struct { + S3Client S3Client +} + +func (c *s3Snapshot) sendToS3(ctx context.Context, key string, bucket string, body io.Reader, cs string) error { if c.sendToS3Func != nil { return c.sendToS3Func(ctx, key, bucket, body) } - ul, err := c.getS3Uploader() + + client, err := c.getS3Client() if err != nil { return err } - output, err := ul.UploadWithContext(ctx, &s3manager.UploadInput{ - Bucket: &bucket, - Key: &key, - Body: body, + + var basics = BucketBasics{ + S3Client: client, + } + var partMiBs int64 = 16 + uploader := manager.NewUploader(basics.S3Client, func(u *manager.Uploader) { + u.PartSize = partMiBs * 1024 * 1024 + }) + + output, err := uploader.Upload(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + Body: body, + ChecksumAlgorithm: "sha256", + Metadata: map[string]string{ + "checksum": cs, + }, }) if err == nil { events.Log("Wrote to S3 location: %s", output.Location) + } else { + events.Log("Couldn't upload s3 snapshot to %v:%v. Here's why: %v\n", + bucket, key, err) } return errors.Wrap(err, "upload with context") } -func (c *s3Snapshot) getS3Uploader() (S3Uploader, error) { - if c.s3Uploader != nil { - return c.s3Uploader, nil +func (c *s3Snapshot) getS3Client() (S3Client, error) { + if c.s3Client != nil { + return c.s3Client, nil } - sess, err := session.NewSession() + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { - return nil, errors.Wrap(err, "creating aws session") + panic(fmt.Sprintf("failed loading config, %v", err)) } - uploader := s3manager.NewUploader(sess) - return uploader, nil + + client := s3.NewFromConfig(cfg) + return client, nil } func archivedSnapshotFromURL(URL string) (archivedSnapshot, error) { diff --git a/pkg/supervisor/fakes/s3_uploader.go b/pkg/supervisor/fakes/s3_uploader.go index 93d0db9d..77aef812 100644 --- a/pkg/supervisor/fakes/s3_uploader.go +++ b/pkg/supervisor/fakes/s3_uploader.go @@ -5,57 +5,236 @@ import ( "context" "sync" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/segmentio/ctlstore/pkg/supervisor" ) -type FakeS3Uploader struct { - UploadStub func(*s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) - uploadMutex sync.RWMutex - uploadArgsForCall []struct { - arg1 *s3manager.UploadInput - arg2 []func(*s3manager.Uploader) +type FakeS3Client struct { + AbortMultipartUploadStub func(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) + abortMultipartUploadMutex sync.RWMutex + abortMultipartUploadArgsForCall []struct { + arg1 context.Context + arg2 *s3.AbortMultipartUploadInput + arg3 []func(*s3.Options) + } + abortMultipartUploadReturns struct { + result1 *s3.AbortMultipartUploadOutput + result2 error + } + abortMultipartUploadReturnsOnCall map[int]struct { + result1 *s3.AbortMultipartUploadOutput + result2 error + } + CompleteMultipartUploadStub func(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) + completeMultipartUploadMutex sync.RWMutex + completeMultipartUploadArgsForCall []struct { + arg1 context.Context + arg2 *s3.CompleteMultipartUploadInput + arg3 []func(*s3.Options) + } + completeMultipartUploadReturns struct { + result1 *s3.CompleteMultipartUploadOutput + result2 error + } + completeMultipartUploadReturnsOnCall map[int]struct { + result1 *s3.CompleteMultipartUploadOutput + result2 error + } + CreateMultipartUploadStub func(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) + createMultipartUploadMutex sync.RWMutex + createMultipartUploadArgsForCall []struct { + arg1 context.Context + arg2 *s3.CreateMultipartUploadInput + arg3 []func(*s3.Options) + } + createMultipartUploadReturns struct { + result1 *s3.CreateMultipartUploadOutput + result2 error } - uploadReturns struct { - result1 *s3manager.UploadOutput + createMultipartUploadReturnsOnCall map[int]struct { + result1 *s3.CreateMultipartUploadOutput result2 error } - uploadReturnsOnCall map[int]struct { - result1 *s3manager.UploadOutput + PutObjectStub func(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) + putObjectMutex sync.RWMutex + putObjectArgsForCall []struct { + arg1 context.Context + arg2 *s3.PutObjectInput + arg3 []func(*s3.Options) + } + putObjectReturns struct { + result1 *s3.PutObjectOutput result2 error } - UploadWithContextStub func(context.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) - uploadWithContextMutex sync.RWMutex - uploadWithContextArgsForCall []struct { + putObjectReturnsOnCall map[int]struct { + result1 *s3.PutObjectOutput + result2 error + } + UploadPartStub func(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error) + uploadPartMutex sync.RWMutex + uploadPartArgsForCall []struct { arg1 context.Context - arg2 *s3manager.UploadInput - arg3 []func(*s3manager.Uploader) + arg2 *s3.UploadPartInput + arg3 []func(*s3.Options) } - uploadWithContextReturns struct { - result1 *s3manager.UploadOutput + uploadPartReturns struct { + result1 *s3.UploadPartOutput result2 error } - uploadWithContextReturnsOnCall map[int]struct { - result1 *s3manager.UploadOutput + uploadPartReturnsOnCall map[int]struct { + result1 *s3.UploadPartOutput result2 error } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeS3Uploader) Upload(arg1 *s3manager.UploadInput, arg2 ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) { - fake.uploadMutex.Lock() - ret, specificReturn := fake.uploadReturnsOnCall[len(fake.uploadArgsForCall)] - fake.uploadArgsForCall = append(fake.uploadArgsForCall, struct { - arg1 *s3manager.UploadInput - arg2 []func(*s3manager.Uploader) - }{arg1, arg2}) - stub := fake.UploadStub - fakeReturns := fake.uploadReturns - fake.recordInvocation("Upload", []interface{}{arg1, arg2}) - fake.uploadMutex.Unlock() +func (fake *FakeS3Client) AbortMultipartUpload(arg1 context.Context, arg2 *s3.AbortMultipartUploadInput, arg3 ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + fake.abortMultipartUploadMutex.Lock() + ret, specificReturn := fake.abortMultipartUploadReturnsOnCall[len(fake.abortMultipartUploadArgsForCall)] + fake.abortMultipartUploadArgsForCall = append(fake.abortMultipartUploadArgsForCall, struct { + arg1 context.Context + arg2 *s3.AbortMultipartUploadInput + arg3 []func(*s3.Options) + }{arg1, arg2, arg3}) + stub := fake.AbortMultipartUploadStub + fakeReturns := fake.abortMultipartUploadReturns + fake.recordInvocation("AbortMultipartUpload", []interface{}{arg1, arg2, arg3}) + fake.abortMultipartUploadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3...) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeS3Client) AbortMultipartUploadCallCount() int { + fake.abortMultipartUploadMutex.RLock() + defer fake.abortMultipartUploadMutex.RUnlock() + return len(fake.abortMultipartUploadArgsForCall) +} + +func (fake *FakeS3Client) AbortMultipartUploadCalls(stub func(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)) { + fake.abortMultipartUploadMutex.Lock() + defer fake.abortMultipartUploadMutex.Unlock() + fake.AbortMultipartUploadStub = stub +} + +func (fake *FakeS3Client) AbortMultipartUploadArgsForCall(i int) (context.Context, *s3.AbortMultipartUploadInput, []func(*s3.Options)) { + fake.abortMultipartUploadMutex.RLock() + defer fake.abortMultipartUploadMutex.RUnlock() + argsForCall := fake.abortMultipartUploadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeS3Client) AbortMultipartUploadReturns(result1 *s3.AbortMultipartUploadOutput, result2 error) { + fake.abortMultipartUploadMutex.Lock() + defer fake.abortMultipartUploadMutex.Unlock() + fake.AbortMultipartUploadStub = nil + fake.abortMultipartUploadReturns = struct { + result1 *s3.AbortMultipartUploadOutput + result2 error + }{result1, result2} +} + +func (fake *FakeS3Client) AbortMultipartUploadReturnsOnCall(i int, result1 *s3.AbortMultipartUploadOutput, result2 error) { + fake.abortMultipartUploadMutex.Lock() + defer fake.abortMultipartUploadMutex.Unlock() + fake.AbortMultipartUploadStub = nil + if fake.abortMultipartUploadReturnsOnCall == nil { + fake.abortMultipartUploadReturnsOnCall = make(map[int]struct { + result1 *s3.AbortMultipartUploadOutput + result2 error + }) + } + fake.abortMultipartUploadReturnsOnCall[i] = struct { + result1 *s3.AbortMultipartUploadOutput + result2 error + }{result1, result2} +} + +func (fake *FakeS3Client) CompleteMultipartUpload(arg1 context.Context, arg2 *s3.CompleteMultipartUploadInput, arg3 ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + fake.completeMultipartUploadMutex.Lock() + ret, specificReturn := fake.completeMultipartUploadReturnsOnCall[len(fake.completeMultipartUploadArgsForCall)] + fake.completeMultipartUploadArgsForCall = append(fake.completeMultipartUploadArgsForCall, struct { + arg1 context.Context + arg2 *s3.CompleteMultipartUploadInput + arg3 []func(*s3.Options) + }{arg1, arg2, arg3}) + stub := fake.CompleteMultipartUploadStub + fakeReturns := fake.completeMultipartUploadReturns + fake.recordInvocation("CompleteMultipartUpload", []interface{}{arg1, arg2, arg3}) + fake.completeMultipartUploadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3...) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeS3Client) CompleteMultipartUploadCallCount() int { + fake.completeMultipartUploadMutex.RLock() + defer fake.completeMultipartUploadMutex.RUnlock() + return len(fake.completeMultipartUploadArgsForCall) +} + +func (fake *FakeS3Client) CompleteMultipartUploadCalls(stub func(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)) { + fake.completeMultipartUploadMutex.Lock() + defer fake.completeMultipartUploadMutex.Unlock() + fake.CompleteMultipartUploadStub = stub +} + +func (fake *FakeS3Client) CompleteMultipartUploadArgsForCall(i int) (context.Context, *s3.CompleteMultipartUploadInput, []func(*s3.Options)) { + fake.completeMultipartUploadMutex.RLock() + defer fake.completeMultipartUploadMutex.RUnlock() + argsForCall := fake.completeMultipartUploadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeS3Client) CompleteMultipartUploadReturns(result1 *s3.CompleteMultipartUploadOutput, result2 error) { + fake.completeMultipartUploadMutex.Lock() + defer fake.completeMultipartUploadMutex.Unlock() + fake.CompleteMultipartUploadStub = nil + fake.completeMultipartUploadReturns = struct { + result1 *s3.CompleteMultipartUploadOutput + result2 error + }{result1, result2} +} + +func (fake *FakeS3Client) CompleteMultipartUploadReturnsOnCall(i int, result1 *s3.CompleteMultipartUploadOutput, result2 error) { + fake.completeMultipartUploadMutex.Lock() + defer fake.completeMultipartUploadMutex.Unlock() + fake.CompleteMultipartUploadStub = nil + if fake.completeMultipartUploadReturnsOnCall == nil { + fake.completeMultipartUploadReturnsOnCall = make(map[int]struct { + result1 *s3.CompleteMultipartUploadOutput + result2 error + }) + } + fake.completeMultipartUploadReturnsOnCall[i] = struct { + result1 *s3.CompleteMultipartUploadOutput + result2 error + }{result1, result2} +} + +func (fake *FakeS3Client) CreateMultipartUpload(arg1 context.Context, arg2 *s3.CreateMultipartUploadInput, arg3 ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + fake.createMultipartUploadMutex.Lock() + ret, specificReturn := fake.createMultipartUploadReturnsOnCall[len(fake.createMultipartUploadArgsForCall)] + fake.createMultipartUploadArgsForCall = append(fake.createMultipartUploadArgsForCall, struct { + arg1 context.Context + arg2 *s3.CreateMultipartUploadInput + arg3 []func(*s3.Options) + }{arg1, arg2, arg3}) + stub := fake.CreateMultipartUploadStub + fakeReturns := fake.createMultipartUploadReturns + fake.recordInvocation("CreateMultipartUpload", []interface{}{arg1, arg2, arg3}) + fake.createMultipartUploadMutex.Unlock() if stub != nil { - return stub(arg1, arg2...) + return stub(arg1, arg2, arg3...) } if specificReturn { return ret.result1, ret.result2 @@ -63,63 +242,129 @@ func (fake *FakeS3Uploader) Upload(arg1 *s3manager.UploadInput, arg2 ...func(*s3 return fakeReturns.result1, fakeReturns.result2 } -func (fake *FakeS3Uploader) UploadCallCount() int { - fake.uploadMutex.RLock() - defer fake.uploadMutex.RUnlock() - return len(fake.uploadArgsForCall) +func (fake *FakeS3Client) CreateMultipartUploadCallCount() int { + fake.createMultipartUploadMutex.RLock() + defer fake.createMultipartUploadMutex.RUnlock() + return len(fake.createMultipartUploadArgsForCall) } -func (fake *FakeS3Uploader) UploadCalls(stub func(*s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)) { - fake.uploadMutex.Lock() - defer fake.uploadMutex.Unlock() - fake.UploadStub = stub +func (fake *FakeS3Client) CreateMultipartUploadCalls(stub func(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)) { + fake.createMultipartUploadMutex.Lock() + defer fake.createMultipartUploadMutex.Unlock() + fake.CreateMultipartUploadStub = stub +} + +func (fake *FakeS3Client) CreateMultipartUploadArgsForCall(i int) (context.Context, *s3.CreateMultipartUploadInput, []func(*s3.Options)) { + fake.createMultipartUploadMutex.RLock() + defer fake.createMultipartUploadMutex.RUnlock() + argsForCall := fake.createMultipartUploadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeS3Uploader) UploadArgsForCall(i int) (*s3manager.UploadInput, []func(*s3manager.Uploader)) { - fake.uploadMutex.RLock() - defer fake.uploadMutex.RUnlock() - argsForCall := fake.uploadArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 +func (fake *FakeS3Client) CreateMultipartUploadReturns(result1 *s3.CreateMultipartUploadOutput, result2 error) { + fake.createMultipartUploadMutex.Lock() + defer fake.createMultipartUploadMutex.Unlock() + fake.CreateMultipartUploadStub = nil + fake.createMultipartUploadReturns = struct { + result1 *s3.CreateMultipartUploadOutput + result2 error + }{result1, result2} +} + +func (fake *FakeS3Client) CreateMultipartUploadReturnsOnCall(i int, result1 *s3.CreateMultipartUploadOutput, result2 error) { + fake.createMultipartUploadMutex.Lock() + defer fake.createMultipartUploadMutex.Unlock() + fake.CreateMultipartUploadStub = nil + if fake.createMultipartUploadReturnsOnCall == nil { + fake.createMultipartUploadReturnsOnCall = make(map[int]struct { + result1 *s3.CreateMultipartUploadOutput + result2 error + }) + } + fake.createMultipartUploadReturnsOnCall[i] = struct { + result1 *s3.CreateMultipartUploadOutput + result2 error + }{result1, result2} +} + +func (fake *FakeS3Client) PutObject(arg1 context.Context, arg2 *s3.PutObjectInput, arg3 ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + fake.putObjectMutex.Lock() + ret, specificReturn := fake.putObjectReturnsOnCall[len(fake.putObjectArgsForCall)] + fake.putObjectArgsForCall = append(fake.putObjectArgsForCall, struct { + arg1 context.Context + arg2 *s3.PutObjectInput + arg3 []func(*s3.Options) + }{arg1, arg2, arg3}) + stub := fake.PutObjectStub + fakeReturns := fake.putObjectReturns + fake.recordInvocation("PutObject", []interface{}{arg1, arg2, arg3}) + fake.putObjectMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3...) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeS3Client) PutObjectCallCount() int { + fake.putObjectMutex.RLock() + defer fake.putObjectMutex.RUnlock() + return len(fake.putObjectArgsForCall) +} + +func (fake *FakeS3Client) PutObjectCalls(stub func(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)) { + fake.putObjectMutex.Lock() + defer fake.putObjectMutex.Unlock() + fake.PutObjectStub = stub +} + +func (fake *FakeS3Client) PutObjectArgsForCall(i int) (context.Context, *s3.PutObjectInput, []func(*s3.Options)) { + fake.putObjectMutex.RLock() + defer fake.putObjectMutex.RUnlock() + argsForCall := fake.putObjectArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeS3Uploader) UploadReturns(result1 *s3manager.UploadOutput, result2 error) { - fake.uploadMutex.Lock() - defer fake.uploadMutex.Unlock() - fake.UploadStub = nil - fake.uploadReturns = struct { - result1 *s3manager.UploadOutput +func (fake *FakeS3Client) PutObjectReturns(result1 *s3.PutObjectOutput, result2 error) { + fake.putObjectMutex.Lock() + defer fake.putObjectMutex.Unlock() + fake.PutObjectStub = nil + fake.putObjectReturns = struct { + result1 *s3.PutObjectOutput result2 error }{result1, result2} } -func (fake *FakeS3Uploader) UploadReturnsOnCall(i int, result1 *s3manager.UploadOutput, result2 error) { - fake.uploadMutex.Lock() - defer fake.uploadMutex.Unlock() - fake.UploadStub = nil - if fake.uploadReturnsOnCall == nil { - fake.uploadReturnsOnCall = make(map[int]struct { - result1 *s3manager.UploadOutput +func (fake *FakeS3Client) PutObjectReturnsOnCall(i int, result1 *s3.PutObjectOutput, result2 error) { + fake.putObjectMutex.Lock() + defer fake.putObjectMutex.Unlock() + fake.PutObjectStub = nil + if fake.putObjectReturnsOnCall == nil { + fake.putObjectReturnsOnCall = make(map[int]struct { + result1 *s3.PutObjectOutput result2 error }) } - fake.uploadReturnsOnCall[i] = struct { - result1 *s3manager.UploadOutput + fake.putObjectReturnsOnCall[i] = struct { + result1 *s3.PutObjectOutput result2 error }{result1, result2} } -func (fake *FakeS3Uploader) UploadWithContext(arg1 context.Context, arg2 *s3manager.UploadInput, arg3 ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) { - fake.uploadWithContextMutex.Lock() - ret, specificReturn := fake.uploadWithContextReturnsOnCall[len(fake.uploadWithContextArgsForCall)] - fake.uploadWithContextArgsForCall = append(fake.uploadWithContextArgsForCall, struct { +func (fake *FakeS3Client) UploadPart(arg1 context.Context, arg2 *s3.UploadPartInput, arg3 ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + fake.uploadPartMutex.Lock() + ret, specificReturn := fake.uploadPartReturnsOnCall[len(fake.uploadPartArgsForCall)] + fake.uploadPartArgsForCall = append(fake.uploadPartArgsForCall, struct { arg1 context.Context - arg2 *s3manager.UploadInput - arg3 []func(*s3manager.Uploader) + arg2 *s3.UploadPartInput + arg3 []func(*s3.Options) }{arg1, arg2, arg3}) - stub := fake.UploadWithContextStub - fakeReturns := fake.uploadWithContextReturns - fake.recordInvocation("UploadWithContext", []interface{}{arg1, arg2, arg3}) - fake.uploadWithContextMutex.Unlock() + stub := fake.UploadPartStub + fakeReturns := fake.uploadPartReturns + fake.recordInvocation("UploadPart", []interface{}{arg1, arg2, arg3}) + fake.uploadPartMutex.Unlock() if stub != nil { return stub(arg1, arg2, arg3...) } @@ -129,58 +374,64 @@ func (fake *FakeS3Uploader) UploadWithContext(arg1 context.Context, arg2 *s3mana return fakeReturns.result1, fakeReturns.result2 } -func (fake *FakeS3Uploader) UploadWithContextCallCount() int { - fake.uploadWithContextMutex.RLock() - defer fake.uploadWithContextMutex.RUnlock() - return len(fake.uploadWithContextArgsForCall) +func (fake *FakeS3Client) UploadPartCallCount() int { + fake.uploadPartMutex.RLock() + defer fake.uploadPartMutex.RUnlock() + return len(fake.uploadPartArgsForCall) } -func (fake *FakeS3Uploader) UploadWithContextCalls(stub func(context.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)) { - fake.uploadWithContextMutex.Lock() - defer fake.uploadWithContextMutex.Unlock() - fake.UploadWithContextStub = stub +func (fake *FakeS3Client) UploadPartCalls(stub func(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error)) { + fake.uploadPartMutex.Lock() + defer fake.uploadPartMutex.Unlock() + fake.UploadPartStub = stub } -func (fake *FakeS3Uploader) UploadWithContextArgsForCall(i int) (context.Context, *s3manager.UploadInput, []func(*s3manager.Uploader)) { - fake.uploadWithContextMutex.RLock() - defer fake.uploadWithContextMutex.RUnlock() - argsForCall := fake.uploadWithContextArgsForCall[i] +func (fake *FakeS3Client) UploadPartArgsForCall(i int) (context.Context, *s3.UploadPartInput, []func(*s3.Options)) { + fake.uploadPartMutex.RLock() + defer fake.uploadPartMutex.RUnlock() + argsForCall := fake.uploadPartArgsForCall[i] return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeS3Uploader) UploadWithContextReturns(result1 *s3manager.UploadOutput, result2 error) { - fake.uploadWithContextMutex.Lock() - defer fake.uploadWithContextMutex.Unlock() - fake.UploadWithContextStub = nil - fake.uploadWithContextReturns = struct { - result1 *s3manager.UploadOutput +func (fake *FakeS3Client) UploadPartReturns(result1 *s3.UploadPartOutput, result2 error) { + fake.uploadPartMutex.Lock() + defer fake.uploadPartMutex.Unlock() + fake.UploadPartStub = nil + fake.uploadPartReturns = struct { + result1 *s3.UploadPartOutput result2 error }{result1, result2} } -func (fake *FakeS3Uploader) UploadWithContextReturnsOnCall(i int, result1 *s3manager.UploadOutput, result2 error) { - fake.uploadWithContextMutex.Lock() - defer fake.uploadWithContextMutex.Unlock() - fake.UploadWithContextStub = nil - if fake.uploadWithContextReturnsOnCall == nil { - fake.uploadWithContextReturnsOnCall = make(map[int]struct { - result1 *s3manager.UploadOutput +func (fake *FakeS3Client) UploadPartReturnsOnCall(i int, result1 *s3.UploadPartOutput, result2 error) { + fake.uploadPartMutex.Lock() + defer fake.uploadPartMutex.Unlock() + fake.UploadPartStub = nil + if fake.uploadPartReturnsOnCall == nil { + fake.uploadPartReturnsOnCall = make(map[int]struct { + result1 *s3.UploadPartOutput result2 error }) } - fake.uploadWithContextReturnsOnCall[i] = struct { - result1 *s3manager.UploadOutput + fake.uploadPartReturnsOnCall[i] = struct { + result1 *s3.UploadPartOutput result2 error }{result1, result2} } -func (fake *FakeS3Uploader) Invocations() map[string][][]interface{} { +func (fake *FakeS3Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.uploadMutex.RLock() - defer fake.uploadMutex.RUnlock() - fake.uploadWithContextMutex.RLock() - defer fake.uploadWithContextMutex.RUnlock() + fake.abortMultipartUploadMutex.RLock() + defer fake.abortMultipartUploadMutex.RUnlock() + fake.completeMultipartUploadMutex.RLock() + defer fake.completeMultipartUploadMutex.RUnlock() + fake.createMultipartUploadMutex.RLock() + defer fake.createMultipartUploadMutex.RUnlock() + fake.putObjectMutex.RLock() + defer fake.putObjectMutex.RUnlock() + fake.uploadPartMutex.RLock() + defer fake.uploadPartMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value @@ -188,7 +439,7 @@ func (fake *FakeS3Uploader) Invocations() map[string][][]interface{} { return copiedInvocations } -func (fake *FakeS3Uploader) recordInvocation(key string, args []interface{}) { +func (fake *FakeS3Client) recordInvocation(key string, args []interface{}) { fake.invocationsMutex.Lock() defer fake.invocationsMutex.Unlock() if fake.invocations == nil { @@ -200,4 +451,4 @@ func (fake *FakeS3Uploader) recordInvocation(key string, args []interface{}) { fake.invocations[key] = append(fake.invocations[key], args) } -var _ supervisor.S3Uploader = new(FakeS3Uploader) +var _ supervisor.S3Client = new(FakeS3Client) diff --git a/pkg/supervisor/s3_uploader.go b/pkg/supervisor/s3_uploader.go index 44663092..601ceacb 100644 --- a/pkg/supervisor/s3_uploader.go +++ b/pkg/supervisor/s3_uploader.go @@ -1,8 +1,10 @@ package supervisor -import "github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface" +import ( + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" +) -//counterfeiter:generate -o fakes/s3_uploader.go . S3Uploader -type S3Uploader interface { - s3manageriface.UploaderAPI +//counterfeiter:generate -o fakes/s3_uploader.go . S3Client +type S3Client interface { + manager.UploadAPIClient } diff --git a/scripts/download.sh b/scripts/download.sh index 500eb7bf..98559c99 100755 --- a/scripts/download.sh +++ b/scripts/download.sh @@ -3,6 +3,10 @@ set -eo pipefail CTLSTORE_BOOTSTRAP_URL=$1 +PREFIX="$(echo $CTLSTORE_BOOTSTRAP_URL | grep :// | sed -e's,^\(.*://\).*,\1,g')" +URL="$(echo $CTLSTORE_BOOTSTRAP_URL | sed -e s,$PREFIX,,g)" +BUCKET="$(echo $URL | grep / | cut -d/ -f1)" +KEY="$(echo $URL | grep / | cut -d/ -f2)" CONCURRENCY=${2:-20} DOWNLOADED="false" COMPRESSED="false" @@ -10,13 +14,32 @@ METRICS="/var/spool/ctlstore/metrics.json" START=$(date +%s) END=$(date +%s) +SHA_START=$(date +%s) +SHA_END=$(date +%s) + +get_head_object() { + head_object=$(aws s3api head-object --bucket "${BUCKET}" --key "${KEY}") + echo "$head_object" +} + if [ ! -f /var/spool/ctlstore/ldb.db ]; then # busybox does not support sub-second resolution START=$(date +%s) mkdir -p /var/spool/ctlstore cd /var/spool/ctlstore - s5cmd -r 0 --log debug cp --concurrency $CONCURRENCY $CTLSTORE_BOOTSTRAP_URL . + + echo "Downloading head object from ${CTLSTORE_BOOTSTRAP_URL}" + head_object=$(get_head_object) + + remote_checksum=$(printf '%s\n' "$head_object" | jq -r '.Metadata.checksum // empty') + echo "Remote checksum in sha1: $remote_checksum" + + remote_version=$(printf '%s\n' "$head_object" | jq -r '.VersionId // empty') + echo "Remote version: $remote_version" + + echo "Downloading snapshot from ${CTLSTORE_BOOTSTRAP_URL} with VersionID: ${remote_version}" + s5cmd -r 0 --log debug cp --version-id $remote_version --concurrency $CONCURRENCY $CTLSTORE_BOOTSTRAP_URL . DOWNLOADED="true" if [[ ${CTLSTORE_BOOTSTRAP_URL: -2} == gz ]]; then @@ -25,6 +48,24 @@ if [ ! -f /var/spool/ctlstore/ldb.db ]; then COMPRESSED="true" fi + SHA_START=$(date +%s) + if [ -z $remote_checksum ]; then + echo "Remote checksum sha1 is null, skipping checksum validation" + else + local_checksum=$(shasum snapshot.db | cut -f1 -d\ | xxd -r -p | base64) + echo "Local snapshot checksum in sha1: $local_checksum" + + if [[ "$local_checksum" == "$remote_checksum" ]]; then + echo "Checksum matches" + else + echo "Checksum does not match" + echo "Failed to download intact snapshot" + exit 1 + fi + fi + SHA_END=$(date +%s) + echo "Local checksum calculation took $(($SHA_END - $SHA_START)) seconds" + mv snapshot.db ldb.db END=$(date +%s) echo "ldb.db ready in $(($END - $START)) seconds"