fix(zambda): Retry Zambda fetch for up to 2 minutes; roll back create (#69)

Author: colin-welch

go.mod

@@ -7,9 +7,11 @@ require (
 	github.com/hashicorp/terraform-plugin-framework v1.16.1
 	github.com/hashicorp/terraform-plugin-go v0.29.0
 	github.com/hashicorp/terraform-plugin-log v0.10.0
+	github.com/stretchr/testify v1.10.0
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
@@ -22,6 +24,7 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/oklog/run v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	golang.org/x/net v0.43.0 // indirect
@@ -30,4 +33,5 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
 	google.golang.org/grpc v1.75.1 // indirect
 	google.golang.org/protobuf v1.36.9 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -27,8 +27,6 @@ github.com/hashicorp/terraform-plugin-framework v1.16.1 h1:1+zwFm3MEqd/0K3YBB2v9
 github.com/hashicorp/terraform-plugin-framework v1.16.1/go.mod h1:0xFOxLy5lRzDTayc4dzK/FakIgBhNf/lC4499R9cV4Y=
 github.com/hashicorp/terraform-plugin-go v0.29.0 h1:1nXKl/nSpaYIUBU1IG/EsDOX0vv+9JxAltQyDMpq5mU=
 github.com/hashicorp/terraform-plugin-go v0.29.0/go.mod h1:vYZbIyvxyy0FWSmDHChCqKvI40cFTDGSb3D8D70i9GM=
-github.com/hashicorp/terraform-plugin-log v0.9.0 h1:i7hOA+vdAItN1/7UrfBqBwvYPQ9TFvymaRGZED3FCV0=
-github.com/hashicorp/terraform-plugin-log v0.9.0/go.mod h1:rKL8egZQ/eXSyDqzLUuwUYLVdlYeamldAHSxjUFADow=
 github.com/hashicorp/terraform-plugin-log v0.10.0 h1:eu2kW6/QBVdN4P3Ju2WiB2W3ObjkAsyfBsL3Wh1fj3g=
 github.com/hashicorp/terraform-plugin-log v0.10.0/go.mod h1:/9RR5Cv2aAbrqcTSdNmY1NRHP4E3ekrXRGjqORpXyB0=
 github.com/hashicorp/terraform-registry-address v0.4.0 h1:S1yCGomj30Sao4l5BMPjTGZmCNzuv7/GDTDX99E9gTk=
@@ -96,6 +94,7 @@ google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
 google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
 google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
 google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/provider/zambda_resource.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/terraform-plugin-framework/attr"
 	"github.com/hashicorp/terraform-plugin-framework/diag"
@@ -250,10 +251,22 @@ func (r *ZambdaResource) Create(ctx context.Context, req resource.CreateRequest,
 	retrievedZambda, err := r.getZambdaAfterMutation(ctx, &resp.Diagnostics, *createdZambda.ID, plan.SourceChecksum.ValueString())
 	if err != nil {
 		resp.Diagnostics.AddError("Error Retrieving Created Zambda", err.Error())
+		// Roll back Zambda create
+		err := r.client.Zambda.DeleteZambda(ctx, *createdZambda.ID)
+		if err != nil {
+			resp.Diagnostics.AddError("Error Deleting Zambda", err.Error())
+			return
+		}
 		return
 	}
 	if retrievedZambda == nil {
 		// Error already added to diagnostics in getZambdaAfterMutation
+		// Roll back Zambda create
+		err := r.client.Zambda.DeleteZambda(ctx, *createdZambda.ID)
+		if err != nil {
+			resp.Diagnostics.AddError("Error Deleting Zambda", err.Error())
+			return
+		}
 		return
 	}
 
@@ -409,8 +422,9 @@ func (r *ZambdaResource) getZambdaAfterMutation(ctx context.Context, diags *diag
 		return retrievedZambda, nil
 	}, retry.RetryConfig{
 		BaseBackoff: retry.BaseBackoffDefault,
-		MaxBackoff:  16000,
-		MaxAttempts: 10,
+		MaxBackoff:  8 * time.Second,
+		MaxDuration: 2 * time.Minute,
+		MaxAttempts: retry.Disabled, // disable max attempts
 	})
 }
 

internal/retry/retry.go

@@ -2,45 +2,62 @@ package retry
 
 import (
 	"context"
+	"math"
 	"math/rand"
 	"time"
 
 	"github.com/hashicorp/terraform-plugin-log/tflog"
 )
 
 const (
-	BaseBackoffDefault time.Duration = 500
-	MaxBackoffDefault  time.Duration = 8000
+	BaseBackoffDefault time.Duration = 500 * time.Millisecond
+	MaxBackoffDefault  time.Duration = 8000 * time.Millisecond
+	MaxDurationDefault time.Duration = 30 * time.Second
 	MaxAttemptsDefault               = 3
+	Disabled                         = 0
 )
 
 var (
 	// DefaultRetryConfig defines the default configuration for retrying operations.
 	DefaultRetryConfig = RetryConfig{
-		BaseBackoff: BaseBackoffDefault,
-		MaxBackoff:  MaxBackoffDefault,
-		MaxAttempts: MaxAttemptsDefault,
+		BaseBackoff:   BaseBackoffDefault,
+		MaxBackoff:    MaxBackoffDefault,
+		MaxDuration:   MaxDurationDefault,
+		MaxAttempts:   MaxAttemptsDefault,
+		DisableJitter: false,
 	}
 )
 
 type RetryConfig struct {
 	BaseBackoff time.Duration
 	MaxBackoff  time.Duration
+	// Set to retry.Disable to disable max duration
+	MaxDuration time.Duration
+	// Set to retry.Disable to disable max attempts
 	MaxAttempts int
+	// Set to true to wait the full backoff
+	DisableJitter bool
 }
 
 func RetryWithBackoff[T any](ctx context.Context, operation func() (T, error), config RetryConfig) (T, error) {
 	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
-	baseBackoff := time.Duration(config.BaseBackoff) * time.Millisecond
-	maxBackoff := time.Duration(config.MaxBackoff) * time.Millisecond
+	baseBackoff := config.BaseBackoff
+	maxBackoff := config.MaxBackoff
+	maxDuration := config.MaxDuration
+	start := time.Now()
 
-	for attempt := range config.MaxAttempts {
+	for attempt := range int(math.Max(float64(config.MaxAttempts), 1000)) {
 		res, err := operation()
 		if err == nil {
 			return res, nil
 		}
 		// If this was the last attempt, return the error
-		if attempt == config.MaxAttempts-1 {
+		if config.MaxAttempts > Disabled && attempt == config.MaxAttempts-1 {
+			var zero T
+			return zero, err
+		}
+		// If we have elapsed max duration, return the error
+		if config.MaxDuration > Disabled && time.Since(start) > maxDuration {
 			var zero T
 			return zero, err
 		}
@@ -50,6 +67,9 @@ func RetryWithBackoff[T any](ctx context.Context, operation func() (T, error), c
 			backoff = maxBackoff
 		}
 		jitter := time.Duration(rng.Int63n(int64(backoff)))
+		if config.DisableJitter {
+			jitter = backoff
+		}
 		tflog.Debug(ctx, "Retrying operation after backoff", map[string]any{
 			"attempt":      attempt + 1,
 			"max_attempts": config.MaxAttempts,

internal/retry/retry_test.go

@@ -0,0 +1,114 @@
+package retry
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRetry(t *testing.T) {
+	tt := []struct {
+		name          string
+		numFailures   int
+		baseBackoff   time.Duration
+		maxBackoff    time.Duration
+		maxDuration   time.Duration
+		maxAttempts   int
+		disableJitter bool
+		expectedRes   int
+		expectedErr   error
+	}{
+		{
+			name:        "defaults success",
+			numFailures: 1,
+			baseBackoff: BaseBackoffDefault,
+			maxBackoff:  MaxBackoffDefault,
+			maxDuration: MaxDurationDefault,
+			maxAttempts: MaxAttemptsDefault,
+			expectedRes: 42,
+			expectedErr: nil,
+		},
+		{
+			name:        "defaults error",
+			numFailures: MaxAttemptsDefault + 1, // 1 more than our attempts
+			baseBackoff: BaseBackoffDefault,
+			maxBackoff:  MaxBackoffDefault,
+			maxDuration: MaxDurationDefault,
+			maxAttempts: MaxAttemptsDefault,
+			expectedRes: 0,
+			expectedErr: fmt.Errorf("some error"),
+		},
+		{
+			name:          "no max duration success",
+			numFailures:   5,                 // 1 less than our attempts
+			baseBackoff:   MaxBackoffDefault, // always 8 seconds wait
+			maxBackoff:    MaxBackoffDefault,
+			maxDuration:   Disabled,
+			maxAttempts:   6,    // 6 tries, 5 waits; wait 5 times is longer than 30 seconds and has a check > 30 seconds
+			disableJitter: true, // wait full time to prove we don't use duration
+			expectedRes:   42,
+			expectedErr:   nil,
+		},
+		{
+			name:          "no max duration error",
+			numFailures:   7,                 // 1 more than our attempts
+			baseBackoff:   MaxBackoffDefault, // always 8 seconds wait
+			maxBackoff:    MaxBackoffDefault,
+			maxDuration:   Disabled,
+			maxAttempts:   6,    // 6 tries, 5 waits; wait 5 times is longer than 30 seconds and has a check > 30 seconds
+			disableJitter: true, // wait full time to prove we don't use duration
+			expectedRes:   0,
+			expectedErr:   fmt.Errorf("some error"),
+		},
+		{
+			name:          "no max attempts success",
+			numFailures:   3,                 // (3 - 1) * 8 = 16 seconds
+			baseBackoff:   MaxBackoffDefault, // always 8 seconds wait
+			maxBackoff:    MaxBackoffDefault,
+			maxDuration:   20 * time.Second,
+			maxAttempts:   Disabled,
+			disableJitter: true, // wait full time to prove we don't use attempts
+			expectedRes:   42,
+			expectedErr:   nil,
+		},
+		{
+			name:          "no max attempts error",
+			numFailures:   4,                 // (4 - 1) * 8 = 24 seconds
+			baseBackoff:   MaxBackoffDefault, // always 8 seconds wait
+			maxBackoff:    MaxBackoffDefault,
+			maxDuration:   20 * time.Second,
+			maxAttempts:   Disabled,
+			disableJitter: true, // wait full time to prove we don't use attempts
+			expectedRes:   0,
+			expectedErr:   fmt.Errorf("some error"),
+		},
+	}
+	wg := sync.WaitGroup{}
+	for _, tc := range tt {
+		wg.Add(1)
+		go t.Run(tc.name, func(t *testing.T) {
+			var i int
+			op := func() (int, error) {
+				if i > tc.numFailures-1 {
+					return 42, nil
+				}
+				i++
+				return 0, fmt.Errorf("some error")
+			}
+			res, err := RetryWithBackoff(t.Context(), op, RetryConfig{
+				BaseBackoff:   tc.baseBackoff,
+				MaxBackoff:    tc.maxBackoff,
+				MaxDuration:   tc.maxDuration,
+				MaxAttempts:   tc.maxAttempts,
+				DisableJitter: tc.disableJitter,
+			})
+			assert.Equal(t, tc.expectedRes, res, "unexpected result")
+			assert.Equal(t, tc.expectedErr, err, "unexpected error")
+			wg.Done()
+		})
+	}
+	wg.Wait()
+}