add backoff options to failover config, did not handle 429 status code logic

Author: glimberea

products/compute/client.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	awsv4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	boff "github.com/cenkalti/backoff/v5"
 )
 
 var DefaultIonosBasePath = ""
@@ -153,6 +154,17 @@ type FailoverOptions struct {
 	// FailoverOnStatusCodes controls whether the transport should fail over to the
 	// next server when it receives one of these HTTP status codes.
 	FailoverOnStatusCodes []int `json:"failoverOnStatusCodes,omitempty" yaml:"failoverOnStatusCodes,omitempty"`
+
+	ExponentialBackoff *ExponentialBackoffOptions `json:"exponentialBackoff,omitempty" yaml:"exponentialBackoff,omitempty"`
+}
+
+// ExponentialBackoffOptions controls the backoff parameters for exponential backoff.
+// It is nested under Configuration so it can be grouped cleanly in JSON/YAML.
+type ExponentialBackoffOptions struct {
+	InitialInterval     time.Duration `json:"initialInterval,omitempty" yaml:"initialInterval,omitempty"`
+	MaxInterval         time.Duration `json:"maxInterval,omitempty" yaml:"maxInterval,omitempty"`
+	Multiplier          float64       `json:"multiplier,omitempty" yaml:"multiplier,omitempty"`
+	RandomizationFactor float64       `json:"randomizationFactor,omitempty" yaml:"randomizationFactor,omitempty"`
 }
 
 // Configuration stores the configuration of the API client
@@ -180,6 +192,31 @@ type Configuration struct {
 	Failover *FailoverOptions `json:"failover,omitempty"`
 }
 
+func (b *ExponentialBackoffOptions) NewExponentialBackoff() *boff.ExponentialBackOff {
+	bo := boff.NewExponentialBackOff()
+	if b == nil {
+		return bo
+	}
+
+	if b.InitialInterval != 0 {
+		bo.InitialInterval = b.InitialInterval
+	}
+
+	if b.MaxInterval != 0 {
+		bo.MaxInterval = b.MaxInterval
+	}
+
+	if b.Multiplier != 0 {
+		bo.Multiplier = b.Multiplier
+	}
+
+	if b.RandomizationFactor != 0 {
+		bo.RandomizationFactor = b.RandomizationFactor
+	}
+
+	return bo
+}
+
 // NewConfiguration returns a new shared.Configuration object.
 // We recommend using NewConfigurationFromOptions, which allows to set more options and initializes the object storage middleware
 func NewConfiguration(username, password, token, hostUrl string) *Configuration {

shared/configuration.go

@@ -34,19 +34,17 @@ import (
 // preserving path and query.
 type FailoverRoundTripper struct {
 	cfg  *Configuration
-	opts *FailoverOptions
 	base http.RoundTripper
 }
 
 // NewFailoverRoundTripper creates a new FailoverRoundTripper.
 // If opts is nil, it will fall back to cfg.Failover.
-func NewFailoverRoundTripper(cfg *Configuration, opts *FailoverOptions, base http.RoundTripper) http.RoundTripper {
+func NewFailoverRoundTripper(cfg *Configuration, base http.RoundTripper) http.RoundTripper {
 	if base == nil {
 		base = http.DefaultTransport
 	}
 	return &FailoverRoundTripper{
 		cfg:  cfg,
-		opts: opts,
 		base: base,
 	}
 }
@@ -68,14 +66,13 @@ func (t *FailoverRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
 		return t.base.RoundTrip(req)
 	}
 
-	fo := t.opts
-	if fo == nil {
-		fo = t.cfg.Failover
-	}
+	fo := t.cfg.Failover
 	if fo == nil {
 		return t.base.RoundTrip(req)
 	}
 
+	bo := fo.ExponentialBackoff.NewExponentialBackoff()
+
 	servers := t.cfg.Servers
 	order := serverOrderFor(fo.Strategy, len(servers))
 	if order == nil {
@@ -88,8 +85,9 @@ func (t *FailoverRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
 		return t.base.RoundTrip(req)
 	}
 
+	maxRetries := t.cfg.MaxRetries
 	var lastErr error
-	for attempt := range len(servers) {
+	for attempt := range maxRetries {
 		serverURL := servers[order(attempt)].URL
 
 		targetURL, err := url.Parse(serverURL)
@@ -111,32 +109,33 @@ func (t *FailoverRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
 		}
 
 		resp, err := t.base.RoundTrip(attemptReq)
-		if err == nil {
-			if shouldFailoverOnStatus(fo, resp.StatusCode) {
-				if SdkLogLevel.Satisfies(Debug) {
-					SdkLogger.Printf("[Failover] status=%d triggers failover to next server", resp.StatusCode)
-				}
-				// Drain/close body to allow connection reuse.
-				if resp.Body != nil {
-					_, _ = io.Copy(io.Discard, resp.Body)
-					_ = resp.Body.Close()
-				}
-				lastErr = fmt.Errorf("failover status: %s", resp.Status)
-				continue
+		if err != nil {
+			lastErr = err
+			retryable := isNetworkErrorRT(attemptReq.Context(), err, fo.RetryOnTimeout)
+			if !retryable {
+				return nil, err
 			}
-			return resp, nil
+			if SdkLogLevel.Satisfies(Debug) {
+				SdkLogger.Printf("[Failover] network error: %v; trying next server", err)
+			}
+
+			backoff(attemptReq.Context(), bo.NextBackOff())
+			continue
 		}
 
-		lastErr = err
-		retryable := isNetworkErrorRT(attemptReq.Context(), err, fo.RetryOnTimeout)
-		if !retryable {
-			return nil, err
+		if !shouldFailoverOnStatus(fo, resp.StatusCode) {
+			return resp, nil
 		}
+
 		if SdkLogLevel.Satisfies(Debug) {
-			SdkLogger.Printf("[Failover] network error: %v; trying next server", err)
+			SdkLogger.Printf("[Failover] status=%d triggers failover to next server", resp.StatusCode)
 		}
-
-		tinyBackoff(attemptReq.Context())
+		// Drain/close body to allow connection reuse.
+		if resp.Body != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			_ = resp.Body.Close()
+		}
+		lastErr = fmt.Errorf("failover status: %s", resp.Status)
 	}
 
 	return nil, lastErr
@@ -184,10 +183,6 @@ var defaultRetryableMethods = map[string]bool{
 }
 
 func isNetworkErrorRT(ctx context.Context, err error, retryOnTimeout bool) bool {
-	if err == nil {
-		return false
-	}
-
 	// 1. Check for standard DNS resolution errors (typed).
 	var dnsErr *net.DNSError
 	if errors.As(err, &dnsErr) {
@@ -225,10 +220,7 @@ func isNetworkErrorRT(ctx context.Context, err error, retryOnTimeout bool) bool
 	return false
 }
 
-// todo replace with a more robust backoff strategy exponential with jitter
-func tinyBackoff(ctx context.Context) {
-	// 10ms is enough to avoid busy-looping while staying responsive.
-	t := 10 * time.Millisecond
+func backoff(ctx context.Context, t time.Duration) {
 	if ctx == nil {
 		time.Sleep(t)
 		return

shared/failover_roundtripper.go

@@ -128,7 +128,8 @@ type FileConfig struct {
 	Environments []Environment `yaml:"environments"`
 	// Failover controls transport-level endpoint failover behaviour.
 	// When set, it is applied to the runtime Configuration via ApplyFailoverToConfiguration.
-	Failover *shared.FailoverOptions `yaml:"failover,omitempty"`
+	Failover           *shared.FailoverOptions           `yaml:"failover,omitempty"`
+	ExponentialBackoff *shared.ExponentialBackoffOptions `yaml:"exponentialBackoff,omitempty"`
 }
 
 // DefaultConfigFileName returns the default file path for the loaded configuration

shared/fileconfiguration/fileconfiguration.go

@@ -4,6 +4,7 @@ go 1.24
 
 require (
 	github.com/aws/aws-sdk-go v1.55.7
+	github.com/cenkalti/backoff/v5 v5.0.3
 	github.com/stretchr/testify v1.10.0
 	gopkg.in/yaml.v3 v3.0.1
 )

shared/go.mod

@@ -1,10 +1,13 @@
 github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
 github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -13,6 +16,7 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=