Skip to content

Commit 88632bb

Browse files
corylanouclaude
corylanou
and
claude
authored
feat(s3): add configurable payload signing and Content-MD5 headers for S3-compatible providers (#842)
Co-authored-by: Claude <[email protected]>
1 parent 7c3a44a commit 88632bb

File tree

9 files changed

+449
-192
lines changed

9 files changed

+449
-192
lines changed

‎.github/workflows/manual-integration-tests.yml‎

Lines changed: 62 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@ on:
3131
required: false
3232
default: false
3333
type: boolean
34+
test_tigris:
35+
description: 'Run Fly.io Tigris integration tests'
36+
required: false
37+
default: false
38+
type: boolean
3439

3540
jobs:
3641
setup:
@@ -61,7 +66,8 @@ jobs:
6166
run: |
6267
if [ "${{ github.event.inputs.test_s3 }}" != "true" ] && \
6368
[ "${{ github.event.inputs.test_gcs }}" != "true" ] && \
64-
[ "${{ github.event.inputs.test_abs }}" != "true" ]; then
69+
[ "${{ github.event.inputs.test_abs }}" != "true" ] && \
70+
[ "${{ github.event.inputs.test_tigris }}" != "true" ]; then
6571
echo "::error::At least one test type must be selected"
6672
exit 1
6773
fi
@@ -74,6 +80,7 @@ jobs:
7480
echo "- S3 (AWS): ${{ github.event.inputs.test_s3 }}" >> $GITHUB_STEP_SUMMARY
7581
echo "- Google Cloud Storage: ${{ github.event.inputs.test_gcs }}" >> $GITHUB_STEP_SUMMARY
7682
echo "- Azure Blob Storage: ${{ github.event.inputs.test_abs }}" >> $GITHUB_STEP_SUMMARY
83+
echo "- Fly.io Tigris: ${{ github.event.inputs.test_tigris }}" >> $GITHUB_STEP_SUMMARY
7784
7885
s3-integration:
7986
name: S3 Integration Tests (AWS)
@@ -96,7 +103,7 @@ jobs:
96103

97104
- run: go install ./cmd/litestream
98105

99-
- run: go test -v ./replica_client_test.go -integration s3
106+
- run: go test -v ./replica_client_test.go -integration -replica-clients=s3
100107
env:
101108
LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }}
102109
LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }}
@@ -118,6 +125,47 @@ jobs:
118125
*.log
119126
test-results/
120127
128+
tigris-integration:
129+
name: Tigris Integration Tests
130+
runs-on: ubuntu-latest
131+
needs: setup
132+
if: github.event.inputs.test_tigris == 'true'
133+
concurrency:
134+
group: integration-test-tigris-manual
135+
cancel-in-progress: false
136+
steps:
137+
- uses: actions/checkout@v4
138+
with:
139+
ref: ${{ needs.setup.outputs.ref }}
140+
141+
- uses: actions/setup-go@v5
142+
with:
143+
go-version-file: "go.mod"
144+
145+
- run: go env
146+
147+
- run: go install ./cmd/litestream
148+
149+
- run: go test -v ./replica_client_test.go -integration -replica-clients=tigris
150+
env:
151+
LITESTREAM_TIGRIS_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_TIGRIS_ACCESS_KEY_ID }}
152+
LITESTREAM_TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_TIGRIS_SECRET_ACCESS_KEY }}
153+
154+
- name: Create test results directory
155+
if: always()
156+
run: |
157+
mkdir -p test-results
158+
echo "Test completed at $(date)" > test-results/tigris-test.log
159+
160+
- name: Upload test results
161+
if: always()
162+
uses: actions/upload-artifact@v4
163+
with:
164+
name: tigris-test-results
165+
path: |
166+
*.log
167+
test-results/
168+
121169
gcs-integration:
122170
name: Google Cloud Storage Integration Tests
123171
runs-on: ubuntu-latest
@@ -145,7 +193,7 @@ jobs:
145193

146194
- run: go install ./cmd/litestream
147195

148-
- run: go test -v ./replica_client_test.go -integration gs
196+
- run: go test -v ./replica_client_test.go -integration -replica-clients=gs
149197
env:
150198
GOOGLE_APPLICATION_CREDENTIALS: /opt/gcp.json
151199
LITESTREAM_GS_BUCKET: litestream-github-workflows
@@ -186,7 +234,7 @@ jobs:
186234

187235
- run: go install ./cmd/litestream
188236

189-
- run: go test -v ./replica_client_test.go -integration abs
237+
- run: go test -v ./replica_client_test.go -integration -replica-clients=abs
190238
env:
191239
LITESTREAM_ABS_ACCOUNT_NAME: ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }}
192240
LITESTREAM_ABS_ACCOUNT_KEY: ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }}
@@ -210,7 +258,7 @@ jobs:
210258
summary:
211259
name: Test Summary
212260
runs-on: ubuntu-latest
213-
needs: [setup, s3-integration, gcs-integration, abs-integration]
261+
needs: [setup, s3-integration, gcs-integration, abs-integration, tigris-integration]
214262
if: always()
215263
steps:
216264
- name: Download all artifacts
@@ -250,6 +298,14 @@ jobs:
250298
echo "⏭️ **Azure Blob Storage:** Skipped" >> $GITHUB_STEP_SUMMARY
251299
fi
252300
301+
if [ "${{ needs.tigris-integration.result }}" == "success" ]; then
302+
echo "✅ **Fly.io Tigris:** Passed" >> $GITHUB_STEP_SUMMARY
303+
elif [ "${{ needs.tigris-integration.result }}" == "failure" ]; then
304+
echo "❌ **Fly.io Tigris:** Failed" >> $GITHUB_STEP_SUMMARY
305+
elif [ "${{ needs.tigris-integration.result }}" == "skipped" ]; then
306+
echo "⏭️ **Fly.io Tigris:** Skipped" >> $GITHUB_STEP_SUMMARY
307+
fi
308+
253309
echo "" >> $GITHUB_STEP_SUMMARY
254310
echo "---" >> $GITHUB_STEP_SUMMARY
255311
echo "**Triggered by:** @${{ github.actor }}" >> $GITHUB_STEP_SUMMARY
@@ -272,6 +328,7 @@ jobs:
272328
if ('${{ needs.s3-integration.result }}' === 'failure') failedJobs.push('S3');
273329
if ('${{ needs.gcs-integration.result }}' === 'failure') failedJobs.push('GCS');
274330
if ('${{ needs.abs-integration.result }}' === 'failure') failedJobs.push('Azure');
331+
if ('${{ needs.tigris-integration.result }}' === 'failure') failedJobs.push('Tigris');
275332
276333
if (failedJobs.length > 0) {
277334
statusEmoji = '❌';

‎abs/replica_client.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,8 @@ func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
275275
}
276276

277277
pager := c.client.NewListBlobsFlatPager(c.Bucket, &azblob.ListBlobsFlatOptions{
278-
Prefix: &prefix,
278+
Prefix: &prefix,
279+
Include: azblob.ListBlobsInclude{Metadata: true},
279280
})
280281

281282
for pager.More() {
@@ -336,7 +337,8 @@ func newLTXFileIterator(ctx context.Context, client *ReplicaClient, level int, s
336337
}
337338

338339
itr.pager = client.client.NewListBlobsFlatPager(client.Bucket, &azblob.ListBlobsFlatOptions{
339-
Prefix: &prefix,
340+
Prefix: &prefix,
341+
Include: azblob.ListBlobsInclude{Metadata: true},
340342
})
341343

342344
return itr

‎cmd/litestream/main.go‎

Lines changed: 68 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -811,15 +811,17 @@ type ReplicaSettings struct {
811811
ValidationInterval *time.Duration `yaml:"validation-interval"`
812812

813813
// S3 settings
814-
AccessKeyID string `yaml:"access-key-id"`
815-
SecretAccessKey string `yaml:"secret-access-key"`
816-
Region string `yaml:"region"`
817-
Bucket string `yaml:"bucket"`
818-
Endpoint string `yaml:"endpoint"`
819-
ForcePathStyle *bool `yaml:"force-path-style"`
820-
SkipVerify bool `yaml:"skip-verify"`
821-
PartSize *ByteSize `yaml:"part-size"`
822-
Concurrency *int `yaml:"concurrency"`
814+
AccessKeyID string `yaml:"access-key-id"`
815+
SecretAccessKey string `yaml:"secret-access-key"`
816+
Region string `yaml:"region"`
817+
Bucket string `yaml:"bucket"`
818+
Endpoint string `yaml:"endpoint"`
819+
ForcePathStyle *bool `yaml:"force-path-style"`
820+
SignPayload *bool `yaml:"sign-payload"`
821+
RequireContentMD5 *bool `yaml:"require-content-md5"`
822+
SkipVerify bool `yaml:"skip-verify"`
823+
PartSize *ByteSize `yaml:"part-size"`
824+
Concurrency *int `yaml:"concurrency"`
823825

824826
// ABS settings
825827
AccountName string `yaml:"account-name"`
@@ -894,6 +896,12 @@ func (rs *ReplicaSettings) SetDefaults(src *ReplicaSettings) {
894896
if rs.ForcePathStyle == nil {
895897
rs.ForcePathStyle = src.ForcePathStyle
896898
}
899+
if rs.SignPayload == nil {
900+
rs.SignPayload = src.SignPayload
901+
}
902+
if rs.RequireContentMD5 == nil {
903+
rs.RequireContentMD5 = src.RequireContentMD5
904+
}
897905
if src.SkipVerify {
898906
rs.SkipVerify = true
899907
}
@@ -1101,6 +1109,14 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
11011109

11021110
bucket, configPath := c.Bucket, c.Path
11031111
region, endpoint, skipVerify := c.Region, c.Endpoint, c.SkipVerify
1112+
signPayload := false
1113+
if v := c.SignPayload; v != nil {
1114+
signPayload = *v
1115+
}
1116+
requireContentMD5 := true
1117+
if v := c.RequireContentMD5; v != nil {
1118+
requireContentMD5 = *v
1119+
}
11041120

11051121
// Use path style if an endpoint is explicitly set. This works because the
11061122
// only service to not use path style is AWS which does not use an endpoint.
@@ -1117,10 +1133,14 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
11171133
}
11181134

11191135
var (
1120-
ubucket string
1121-
uregion string
1122-
uendpoint string
1123-
uforcePathStyle bool
1136+
ubucket string
1137+
uregion string
1138+
uendpoint string
1139+
uforcePathStyle bool
1140+
usignPayload bool
1141+
usignPayloadSet bool
1142+
urequireContentMD5 bool
1143+
urequireContentMD5Set bool
11241144
)
11251145

11261146
if strings.HasPrefix(host, "arn:") {
@@ -1152,6 +1172,14 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
11521172
if qSkipVerify := query.Get("skipVerify"); qSkipVerify != "" {
11531173
skipVerify = qSkipVerify == "true"
11541174
}
1175+
if v, ok := boolQueryValue(query, "signPayload", "sign-payload"); ok {
1176+
usignPayload = v
1177+
usignPayloadSet = true
1178+
}
1179+
if v, ok := boolQueryValue(query, "requireContentMD5", "require-content-md5"); ok {
1180+
urequireContentMD5 = v
1181+
urequireContentMD5Set = true
1182+
}
11551183

11561184
// Only apply URL parts to field that have not been overridden.
11571185
if configPath == "" {
@@ -1169,6 +1197,12 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
11691197
if !forcePathStyle {
11701198
forcePathStyle = uforcePathStyle
11711199
}
1200+
if c.SignPayload == nil && usignPayloadSet {
1201+
signPayload = usignPayload
1202+
}
1203+
if c.RequireContentMD5 == nil && urequireContentMD5Set {
1204+
requireContentMD5 = urequireContentMD5
1205+
}
11721206
}
11731207

11741208
// Ensure required settings are set.
@@ -1186,6 +1220,8 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
11861220
client.Endpoint = endpoint
11871221
client.ForcePathStyle = forcePathStyle
11881222
client.SkipVerify = skipVerify
1223+
client.SignPayload = signPayload
1224+
client.RequireContentMD5 = requireContentMD5
11891225

11901226
// Apply upload configuration if specified.
11911227
if c.PartSize != nil {
@@ -1570,6 +1606,25 @@ func cleanReplicaURLPath(p string) string {
15701606
return cleaned
15711607
}
15721608

1609+
func boolQueryValue(query url.Values, keys ...string) (bool, bool) {
1610+
if query == nil {
1611+
return false, false
1612+
}
1613+
for _, key := range keys {
1614+
if raw := query.Get(key); raw != "" {
1615+
switch strings.ToLower(raw) {
1616+
case "true", "1", "t", "yes":
1617+
return true, true
1618+
case "false", "0", "f", "no":
1619+
return false, true
1620+
default:
1621+
return false, true
1622+
}
1623+
}
1624+
}
1625+
return false, false
1626+
}
1627+
15731628
func regionFromS3ARN(arn string) string {
15741629
parts := strings.SplitN(arn, ":", 6)
15751630
if len(parts) >= 4 {

‎cmd/litestream/main_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2250,6 +2250,46 @@ func TestNewS3ReplicaClientFromConfig(t *testing.T) {
22502250
t.Error("expected ForcePathStyle to be true for custom endpoint")
22512251
}
22522252
})
2253+
2254+
t.Run("QuerySigningOptions", func(t *testing.T) {
2255+
config := &main.ReplicaConfig{
2256+
URL: "s3://bucket/db?sign-payload=true&require-content-md5=false",
2257+
}
2258+
2259+
client, err := main.NewS3ReplicaClientFromConfig(config, nil)
2260+
if err != nil {
2261+
t.Fatal(err)
2262+
}
2263+
if !client.SignPayload {
2264+
t.Error("expected SignPayload to be true when query parameter is set")
2265+
}
2266+
if client.RequireContentMD5 {
2267+
t.Error("expected RequireContentMD5 to be false when disabled via query")
2268+
}
2269+
})
2270+
2271+
t.Run("ConfigOverridesQuerySigning", func(t *testing.T) {
2272+
signTrue := true
2273+
requireFalse := false
2274+
config := &main.ReplicaConfig{
2275+
URL: "s3://bucket/db?sign-payload=false&require-content-md5=true",
2276+
ReplicaSettings: main.ReplicaSettings{
2277+
SignPayload: &signTrue,
2278+
RequireContentMD5: &requireFalse,
2279+
},
2280+
}
2281+
2282+
client, err := main.NewS3ReplicaClientFromConfig(config, nil)
2283+
if err != nil {
2284+
t.Fatal(err)
2285+
}
2286+
if !client.SignPayload {
2287+
t.Error("expected config SignPayload to override query parameter")
2288+
}
2289+
if client.RequireContentMD5 {
2290+
t.Error("expected config RequireContentMD5=false to override query parameter")
2291+
}
2292+
})
22532293
}
22542294
func TestGlobalDefaults(t *testing.T) {
22552295
// Test comprehensive global defaults functionality

‎etc/litestream.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
# replica:
88
# path: /path/to/replica # File-based replication
99
# url: s3://my.bucket.com/db # S3-based replication
10+
# # Example Fly.io Tigris setup (requires signed payloads & no Content-MD5 deletes)
11+
# # url: s3://my-tigris-bucket/db.sqlite?endpoint=fly.storage.tigris.dev&region=auto
12+
# # sign-payload: true
13+
# # require-content-md5: false
1014
# type: nats # NATS JetStream replication
1115
# url: nats://nats.example.com:4222
1216
# bucket: litestream-backups

0 commit comments

Comments
 (0)