feat(s3): add automatic Tigris endpoint detection (#845)

Co-authored-by: Claude <[email protected]>

Author: corylanou

cmd/litestream/main.go

@@ -1109,13 +1109,13 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
 
 	bucket, configPath := c.Bucket, c.Path
 	region, endpoint, skipVerify := c.Region, c.Endpoint, c.SkipVerify
-	signPayload := false
+	signSetting := newBoolSetting(false)
 	if v := c.SignPayload; v != nil {
-		signPayload = *v
+		signSetting.Set(*v)
 	}
-	requireContentMD5 := true
+	requireSetting := newBoolSetting(true)
 	if v := c.RequireContentMD5; v != nil {
-		requireContentMD5 = *v
+		requireSetting.Set(*v)
 	}
 
 	// Use path style if an endpoint is explicitly set. This works because the
@@ -1126,21 +1126,28 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
 	}
 
 	// Apply settings from URL, if specified.
+	var (
+		endpointWasSet        bool
+		usignPayload          bool
+		usignPayloadSet       bool
+		urequireContentMD5    bool
+		urequireContentMD5Set bool
+	)
+	if endpoint != "" {
+		endpointWasSet = true
+	}
+
 	if c.URL != "" {
 		_, host, upath, query, err := ParseReplicaURLWithQuery(c.URL)
 		if err != nil {
 			return nil, err
 		}
 
 		var (
-			ubucket               string
-			uregion               string
-			uendpoint             string
-			uforcePathStyle       bool
-			usignPayload          bool
-			usignPayloadSet       bool
-			urequireContentMD5    bool
-			urequireContentMD5Set bool
+			ubucket         string
+			uregion         string
+			uendpoint       string
+			uforcePathStyle bool
 		)
 
 		if strings.HasPrefix(host, "arn:") {
@@ -1162,6 +1169,7 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
 			if query.Get("forcePathStyle") != "false" {
 				uforcePathStyle = true
 			}
+			endpointWasSet = true
 		}
 		if qRegion := query.Get("region"); qRegion != "" {
 			uregion = qRegion
@@ -1197,11 +1205,11 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
 		if !forcePathStyle {
 			forcePathStyle = uforcePathStyle
 		}
-		if c.SignPayload == nil && usignPayloadSet {
-			signPayload = usignPayload
+		if !signSetting.set && usignPayloadSet {
+			signSetting.Set(usignPayload)
 		}
-		if c.RequireContentMD5 == nil && urequireContentMD5Set {
-			requireContentMD5 = urequireContentMD5
+		if !requireSetting.set && urequireContentMD5Set {
+			requireSetting.Set(urequireContentMD5)
 		}
 	}
 
@@ -1210,6 +1218,11 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
 		return nil, fmt.Errorf("bucket required for s3 replica")
 	}
 
+	isTigris := isTigrisEndpoint(endpoint)
+	if !isTigris && !endpointWasSet && isTigrisEndpoint(c.Endpoint) {
+		isTigris = true
+	}
+
 	// Build replica.
 	client := s3.NewReplicaClient()
 	client.AccessKeyID = c.AccessKeyID
@@ -1220,8 +1233,13 @@ func NewS3ReplicaClientFromConfig(c *ReplicaConfig, _ *litestream.Replica) (_ *s
 	client.Endpoint = endpoint
 	client.ForcePathStyle = forcePathStyle
 	client.SkipVerify = skipVerify
-	client.SignPayload = signPayload
-	client.RequireContentMD5 = requireContentMD5
+	if isTigris {
+		signSetting.ApplyDefault(true)
+		requireSetting.ApplyDefault(false)
+	}
+
+	client.SignPayload = signSetting.value
+	client.RequireContentMD5 = requireSetting.value
 
 	// Apply upload configuration if specified.
 	if c.PartSize != nil {
@@ -1625,6 +1643,39 @@ func boolQueryValue(query url.Values, keys ...string) (bool, bool) {
 	return false, false
 }
 
+func isTigrisEndpoint(endpoint string) bool {
+	endpoint = strings.TrimSpace(strings.ToLower(endpoint))
+	if endpoint == "" {
+		return false
+	}
+	if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
+		if u, err := url.Parse(endpoint); err == nil && u.Host != "" {
+			endpoint = u.Host
+		}
+	}
+	return endpoint == "fly.storage.tigris.dev"
+}
+
+type boolSetting struct {
+	value bool
+	set   bool
+}
+
+func newBoolSetting(defaultValue bool) boolSetting {
+	return boolSetting{value: defaultValue}
+}
+
+func (s *boolSetting) Set(value bool) {
+	s.value = value
+	s.set = true
+}
+
+func (s *boolSetting) ApplyDefault(value bool) {
+	if !s.set {
+		s.value = value
+	}
+}
+
 func regionFromS3ARN(arn string) string {
 	parts := strings.SplitN(arn, ":", 6)
 	if len(parts) >= 4 {

cmd/litestream/main_test.go

@@ -2231,6 +2231,34 @@ func TestNewS3ReplicaClientFromConfig(t *testing.T) {
 		if !client.ForcePathStyle {
 			t.Error("expected ForcePathStyle to be true for custom endpoint")
 		}
+		if !client.SignPayload {
+			t.Error("expected SignPayload to be true for Tigris")
+		}
+		if client.RequireContentMD5 {
+			t.Error("expected RequireContentMD5 to be false for Tigris")
+		}
+	})
+
+	t.Run("TigrisConfigEndpoint", func(t *testing.T) {
+		config := &main.ReplicaConfig{
+			Path: "path",
+			ReplicaSettings: main.ReplicaSettings{
+				Bucket:   "mybucket",
+				Endpoint: "https://fly.storage.tigris.dev",
+			},
+		}
+
+		client, err := main.NewS3ReplicaClientFromConfig(config, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !client.SignPayload {
+			t.Error("expected SignPayload to be true for config-based Tigris endpoint")
+		}
+		if client.RequireContentMD5 {
+			t.Error("expected RequireContentMD5 to be false for config-based Tigris endpoint")
+		}
 	})
 
 	t.Run("HTTPSEndpoint", func(t *testing.T) {
@@ -2290,6 +2318,29 @@ func TestNewS3ReplicaClientFromConfig(t *testing.T) {
 			t.Error("expected config RequireContentMD5=false to override query parameter")
 		}
 	})
+
+	t.Run("TigrisManualOverride", func(t *testing.T) {
+		signFalse := false
+		requireTrue := true
+		config := &main.ReplicaConfig{
+			URL: "s3://bucket/db?endpoint=fly.storage.tigris.dev&region=auto",
+			ReplicaSettings: main.ReplicaSettings{
+				SignPayload:       &signFalse,
+				RequireContentMD5: &requireTrue,
+			},
+		}
+
+		client, err := main.NewS3ReplicaClientFromConfig(config, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if client.SignPayload {
+			t.Error("expected manual SignPayload override to take precedence")
+		}
+		if !client.RequireContentMD5 {
+			t.Error("expected manual RequireContentMD5 override to take precedence")
+		}
+	})
 }
 func TestGlobalDefaults(t *testing.T) {
 	// Test comprehensive global defaults functionality

etc/litestream.yml

@@ -7,10 +7,8 @@
 #    replica:
 #      path: /path/to/replica             # File-based replication
 #      url:  s3://my.bucket.com/db        # S3-based replication
-#      # Example Fly.io Tigris setup (requires signed payloads & no Content-MD5 deletes)
+#      # Example Fly.io Tigris setup (signing/no-Content-MD5 are auto-enabled for this endpoint)
 #      # url: s3://my-tigris-bucket/db.sqlite?endpoint=fly.storage.tigris.dev&region=auto
-#      # sign-payload: true
-#      # require-content-md5: false
 #      type: nats                         # NATS JetStream replication
 #      url: nats://nats.example.com:4222
 #      bucket: litestream-backups