Skip to content

Commit 063dab6

Browse files
vecerajsandraromanchenko
veceraj
and
sandraromanchenko
authored
PBM-1498 HMAC support for GCS (#1123)
* create new interface * move implementation to google_client * use interface implementation * hide the credentials * go mod minio * update vendor * updates for multipart upload * enable retryer config * refactor part size --------- Co-authored-by: Sandra Romanchenko <[email protected]>
1 parent a77de80 commit 063dab6

File tree

299 files changed

+70558
-214
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

299 files changed

+70558
-214
lines changed

‎e2e-tests/cmd/pbm-test/run.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ func run(t *sharded.Cluster, typ testTyp) {
2626
}{
2727
{"AWS", "/etc/pbm/aws.yaml"},
2828
{"GCS", "/etc/pbm/gcs.yaml"},
29+
{"GCS_HMAC", "/etc/pbm/gcs_hmac.yaml"},
2930
{"Azure", "/etc/pbm/azure.yaml"},
3031
{"FS", "/etc/pbm/fs.yaml"},
3132
}

‎e2e-tests/cmd/pbm-test/run_physical.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ func runPhysical(t *sharded.Cluster, typ testTyp) {
1717
}{
1818
{"AWS", "/etc/pbm/aws.yaml"},
1919
{"GCS", "/etc/pbm/gcs.yaml"},
20+
{"GCS_HMAC", "/etc/pbm/gcs_hmac.yaml"},
2021
{"Azure", "/etc/pbm/azure.yaml"},
2122
{"FS", "/etc/pbm/fs.yaml"},
2223
}

‎go.mod

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ require (
2323
github.com/googleapis/gax-go/v2 v2.12.3
2424
github.com/klauspost/compress v1.17.11
2525
github.com/klauspost/pgzip v1.2.6
26+
github.com/minio/minio-go/v7 v7.0.68
2627
github.com/mongodb/mongo-tools v0.0.0-20240723193119-837c2bc263f4
2728
github.com/pierrec/lz4 v2.6.1+incompatible
2829
github.com/pkg/errors v0.9.1
@@ -67,6 +68,7 @@ require (
6768
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
6869
github.com/distribution/reference v0.6.0 // indirect
6970
github.com/docker/go-units v0.5.0 // indirect
71+
github.com/dustin/go-humanize v1.0.1 // indirect
7072
github.com/felixge/httpsnoop v1.0.4 // indirect
7173
github.com/go-logr/logr v1.4.1 // indirect
7274
github.com/go-logr/stdr v1.2.2 // indirect
@@ -79,21 +81,28 @@ require (
7981
github.com/hashicorp/hcl v1.0.0 // indirect
8082
github.com/inconshreveable/mousetrap v1.1.0 // indirect
8183
github.com/jessevdk/go-flags v1.5.0 // indirect
84+
github.com/json-iterator/go v1.1.12 // indirect
85+
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
8286
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
8387
github.com/magiconair/properties v1.8.7 // indirect
88+
github.com/minio/md5-simd v1.1.2 // indirect
89+
github.com/minio/sha256-simd v1.0.1 // indirect
8490
github.com/mitchellh/mapstructure v1.5.0 // indirect
8591
github.com/moby/docker-image-spec v1.3.1 // indirect
8692
github.com/moby/patternmatcher v0.6.0 // indirect
8793
github.com/moby/sys/sequential v0.5.0 // indirect
8894
github.com/moby/sys/user v0.1.0 // indirect
8995
github.com/moby/term v0.5.0 // indirect
96+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
97+
github.com/modern-go/reflect2 v1.0.2 // indirect
9098
github.com/montanaflynn/stats v0.7.1 // indirect
9199
github.com/morikuni/aec v1.0.0 // indirect
92100
github.com/opencontainers/go-digest v1.0.0 // indirect
93101
github.com/opencontainers/image-spec v1.1.0 // indirect
94102
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
95103
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
96104
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
105+
github.com/rs/xid v1.5.0 // indirect
97106
github.com/sagikazarmark/locafero v0.4.0 // indirect
98107
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
99108
github.com/shirou/gopsutil/v3 v3.23.12 // indirect

‎go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
152152
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
153153
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
154154
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
155+
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
155156
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
156157
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
157158
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
@@ -181,6 +182,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
181182
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
182183
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
183184
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
185+
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
184186
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
185187
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
186188
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
@@ -213,6 +215,7 @@ github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
213215
github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
214216
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
215217
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
218+
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
216219
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
217220
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
218221
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
@@ -392,6 +395,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
392395
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
393396
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
394397
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
398+
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
395399
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
396400
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
397401
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

‎pbm/config/config.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,12 @@ func (c *Config) String() string {
153153
if c.Storage.GCS.Credentials.ClientEmail != "" {
154154
c.Storage.GCS.Credentials.ClientEmail = "***"
155155
}
156+
if c.Storage.GCS.Credentials.HMACAccessKey != "" {
157+
c.Storage.GCS.Credentials.HMACAccessKey = "***"
158+
}
159+
if c.Storage.GCS.Credentials.HMACSecret != "" {
160+
c.Storage.GCS.Credentials.HMACSecret = "***"
161+
}
156162
}
157163

158164
b, err := yaml.Marshal(c)

‎pbm/storage/gcs/download.go

Lines changed: 5 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,8 @@ import (
55
"context"
66
"io"
77
"net/http"
8-
"path"
98
"time"
109

11-
gcs "cloud.google.com/go/storage"
1210
"google.golang.org/api/googleapi"
1311

1412
"github.com/percona/percona-backup-mongodb/pbm/errors"
@@ -68,14 +66,13 @@ func (g *GCS) newPartReader(fname string, fsize int64, chunkSize int) *storage.P
6866
Buf: make([]byte, 32*1024),
6967
L: g.log,
7068
GetChunk: func(fname string, arena *storage.Arena, cli interface{}, start, end int64) (io.ReadCloser, error) {
71-
bucketHandle, ok := cli.(*gcs.BucketHandle)
72-
if !ok {
73-
return nil, errors.Errorf("expected *gcs.BucketHandle, got %T", cli)
74-
}
75-
return g.getChunk(fname, arena, bucketHandle, start, end)
69+
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
70+
defer cancel()
71+
72+
return cli.(gcsClient).getPartialObject(ctx, fname, start, end-start+1)
7673
},
7774
GetSess: func() (interface{}, error) {
78-
return g.bucketHandle, nil
75+
return g.client, nil // re-use the already-initialized client
7976
},
8077
}
8178
}
@@ -153,37 +150,6 @@ func (g *GCS) sourceReader(fname string, arenas []*storage.Arena, cc, downloadCh
153150
return r, nil
154151
}
155152

156-
func (g *GCS) getChunk(
157-
fname string,
158-
buf *storage.Arena,
159-
bucketHandle *gcs.BucketHandle,
160-
start, end int64,
161-
) (io.ReadCloser, error) {
162-
ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
163-
defer cancel()
164-
165-
length := end - start + 1
166-
obj := bucketHandle.Object(path.Join(g.opts.Prefix, fname))
167-
reader, err := obj.NewRangeReader(ctx, start, length)
168-
if err != nil {
169-
if errors.Is(err, gcs.ErrObjectNotExist) || (err != nil && isRangeNotSatisfiable(err)) {
170-
return nil, io.EOF
171-
}
172-
173-
g.log.Warning("errGetObj Err: %v", err)
174-
return nil, storage.GetObjError{Err: err}
175-
}
176-
177-
ch := buf.GetSpan()
178-
_, err = io.CopyBuffer(ch, reader, buf.CpBuf)
179-
if err != nil {
180-
ch.Close()
181-
return nil, errors.Wrap(err, "copy")
182-
}
183-
reader.Close()
184-
return ch, nil
185-
}
186-
187153
func isRangeNotSatisfiable(err error) bool {
188154
var herr *googleapi.Error
189155
if errors.As(err, &herr) {

0 commit comments

Comments
 (0)