Skip to content

Commit ad7303a

Browse files
overcatovercat
committed
support/datastore: add implementation for S3 and S3-compatible data storage.
1 parent a356b55 commit ad7303a

File tree

10 files changed

+775
-16
lines changed

10 files changed

+775
-16
lines changed

‎go.mod‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ require (
5656
)
5757

5858
require (
59+
github.com/aws/aws-sdk-go-v2 v1.36.5
60+
github.com/aws/aws-sdk-go-v2/config v1.29.17
61+
github.com/aws/aws-sdk-go-v2/credentials v1.17.70
62+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83
63+
github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0
64+
github.com/aws/smithy-go v1.22.4
5965
github.com/cenkalti/backoff/v4 v4.3.0
6066
github.com/creachadair/jrpc2 v1.2.0
6167
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da
@@ -73,6 +79,19 @@ require (
7379
cloud.google.com/go/longrunning v0.5.7 // indirect
7480
cloud.google.com/go/pubsub v1.38.0 // indirect
7581
github.com/andybalholm/brotli v1.0.4 // indirect
82+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
83+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
84+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
85+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
86+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
87+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect
88+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
89+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect
90+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
91+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect
92+
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
93+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
94+
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
7695
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
7796
github.com/cespare/xxhash/v2 v2.2.0 // indirect
7897
github.com/containerd/log v0.1.0 // indirect

‎go.sum‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W
8484
github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
8585
github.com/aws/aws-sdk-go v1.45.27 h1:b+zOTPkAG4i2RvqPdHxkJZafmhhVaVHBp4r41Tu4I6U=
8686
github.com/aws/aws-sdk-go v1.45.27/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
87+
github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0=
88+
github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0=
89+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA=
90+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY=
91+
github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0=
92+
github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8=
93+
github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0=
94+
github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc=
95+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU=
96+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8=
97+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 h1:08otkOELsIi0toRRGMytlJhOctcN8xfKfKFR2NXz3kE=
98+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83/go.mod h1:dGsGb2wI8JDWeMAhjVPP+z+dqvYjL6k6o+EujcRNk5c=
99+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s=
100+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo=
101+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM=
102+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8=
103+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
104+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
105+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs=
106+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw=
107+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc=
108+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ=
109+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074=
110+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE=
111+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI=
112+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4=
113+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4=
114+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U=
115+
github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE=
116+
github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU=
117+
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg=
118+
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc=
119+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg=
120+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E=
121+
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0=
122+
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w=
123+
github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw=
124+
github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
87125
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
88126
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
89127
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=

‎services/galexie/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
All notable changes to this project will be documented in this
44
file. This project adheres to [Semantic Versioning](http://semver.org/).
55

6+
## Unreleased
7+
8+
- Implement S3 and S3-compatible data storage.
9+
610
## [v1.0.0]
711

812
- 🎉 First release!

‎services/galexie/config.example.toml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,23 @@ admin_port = 6061
77

88
# Datastore Configuration
99
[datastore_config]
10-
# Specifies the type of datastore. Currently, only Google Cloud Storage (GCS) is supported.
10+
# Specifies the type of datastore. Currently, Google Cloud Storage (GCS) and s3-compatible storage (S3) are supported.
1111
type = "GCS"
1212

1313
[datastore_config.params]
14+
# params required for GCS storage
1415
# The Google Cloud Storage bucket path for storing data, with optional subpaths for organization.
1516
destination_bucket_path = "your-bucket-name/<optional_subpath1>/<optional_subpath2>/"
1617

18+
# params required for S3-compatible storage
19+
# Check https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#the-shared-credentials-file for more information on how to set up credentials.
20+
# The S3 bucket path for storing data, with optional subpaths for organization.
21+
#destination_bucket_path = "your-bucket-name/<optional_subpath1>/<optional_subpath2>/"
22+
# The region where the S3 bucket is located.
23+
#region = "us-west-1" # Example region, change as needed.
24+
# The endpoint URL for the S3-compatible storage.
25+
#endpoint_url = "https://s3.us-west-1.amazonaws.com" # Example endpoint, change as needed.
26+
1727
[datastore_config.schema]
1828
# Configuration for data organization
1929
ledgers_per_file = 1 # Number of ledgers stored in each file.

‎support/datastore/datastore.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,10 @@ type DataStore interface {
2929
func NewDataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error) {
3030
switch datastoreConfig.Type {
3131
case "GCS":
32-
destinationBucketPath, ok := datastoreConfig.Params["destination_bucket_path"]
33-
if !ok {
34-
return nil, errors.Errorf("Invalid GCS config, no destination_bucket_path")
35-
}
36-
return NewGCSDataStore(ctx, destinationBucketPath, datastoreConfig.Schema)
32+
return NewGCSDataStore(ctx, datastoreConfig)
33+
case "S3":
34+
return NewS3DataStore(ctx, datastoreConfig)
35+
3736
default:
3837
return nil, errors.Errorf("Invalid datastore type %v, not supported", datastoreConfig.Type)
3938
}

‎support/datastore/gcs_datastore.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,18 @@ type GCSDataStore struct {
2727
schema DataStoreSchema
2828
}
2929

30-
func NewGCSDataStore(ctx context.Context, bucketPath string, schema DataStoreSchema) (DataStore, error) {
30+
func NewGCSDataStore(ctx context.Context, dataStoreConfig DataStoreConfig) (DataStore, error) {
31+
destinationBucketPath, ok := dataStoreConfig.Params["destination_bucket_path"]
32+
if !ok {
33+
return nil, errors.New("invalid GCS config, no destination_bucket_path")
34+
}
35+
3136
client, err := storage.NewClient(ctx)
3237
if err != nil {
3338
return nil, err
3439
}
3540

36-
return FromGCSClient(ctx, client, bucketPath, schema)
41+
return FromGCSClient(ctx, client, destinationBucketPath, dataStoreConfig.Schema)
3742
}
3843

3944
func FromGCSClient(ctx context.Context, client *storage.Client, bucketPath string, schema DataStoreSchema) (DataStore, error) {

‎support/datastore/gcs_test.go‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -377,11 +377,3 @@ func TestGCSGetFileValidatesCRC32C(t *testing.T) {
377377
_, err = io.Copy(&buf, reader)
378378
require.EqualError(t, err, "storage: bad CRC on read: got 985946173, want 2601510353")
379379
}
380-
381-
func requireReaderContentEquals(t *testing.T, reader io.ReadCloser, expected []byte) {
382-
var buf bytes.Buffer
383-
_, err := io.Copy(&buf, reader)
384-
require.NoError(t, err)
385-
require.NoError(t, reader.Close())
386-
require.Equal(t, expected, buf.Bytes())
387-
}

‎support/datastore/helpers_test.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package datastore
2+
3+
import (
4+
"bytes"
5+
"github.com/stretchr/testify/require"
6+
"io"
7+
"testing"
8+
)
9+
10+
// requireReaderContentEquals is a helper function to assert reader content.
11+
func requireReaderContentEquals(t *testing.T, reader io.ReadCloser, expected []byte) {
12+
var buf bytes.Buffer
13+
_, err := io.Copy(&buf, reader)
14+
require.NoError(t, err)
15+
require.NoError(t, reader.Close())
16+
require.Equal(t, expected, buf.Bytes())
17+
}

0 commit comments

Comments
 (0)