fix(setting): add retries on creation (#130)

* fix(setting): add retries on creation

* fix: support configurable timeout

Author: nunoadrego

docs/resources/setting.md

@@ -38,5 +38,13 @@ This field must be left null when using a ClickHouse Cloud cluster.
 When using a self hosted ClickHouse instance, this field should only be set when there is more than one replica and you are not using 'replicated' storage for user_directory.
 - `max` (String) Max Value for the setting
 - `min` (String) Min Value for the setting
+- `timeouts` (Attributes) (see [below for nested schema](#nestedatt--timeouts))
 - `value` (String) Value for the setting
 - `writability` (String) Writability attribute for the setting
+
+<a id="nestedatt--timeouts"></a>
+### Nested Schema for `timeouts`
+
+Optional:
+
+- `create` (String) A string that can be [parsed as a duration](https://pkg.go.dev/time#ParseDuration) consisting of numbers and unit suffixes, such as "30s" or "2h45m". Valid time units are "s" (seconds), "m" (minutes), "h" (hours).

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.24.0
 	github.com/hashicorp/terraform-json v0.27.2
 	github.com/hashicorp/terraform-plugin-framework v1.16.1
+	github.com/hashicorp/terraform-plugin-framework-timeouts v0.7.0
 	github.com/hashicorp/terraform-plugin-framework-validators v0.19.0
 	github.com/hashicorp/terraform-plugin-go v0.29.0
 	github.com/hashicorp/terraform-plugin-log v0.10.0

go.sum

@@ -97,6 +97,8 @@ github.com/hashicorp/terraform-json v0.27.2 h1:BwGuzM6iUPqf9JYM/Z4AF1OJ5VVJEEzoK
 github.com/hashicorp/terraform-json v0.27.2/go.mod h1:GzPLJ1PLdUG5xL6xn1OXWIjteQRT2CNT9o/6A9mi9hE=
 github.com/hashicorp/terraform-plugin-framework v1.16.1 h1:1+zwFm3MEqd/0K3YBB2v9u9DtyYHyEuhVOfeIXbteWA=
 github.com/hashicorp/terraform-plugin-framework v1.16.1/go.mod h1:0xFOxLy5lRzDTayc4dzK/FakIgBhNf/lC4499R9cV4Y=
+github.com/hashicorp/terraform-plugin-framework-timeouts v0.7.0 h1:jblRy1PkLfPm5hb5XeMa3tezusnMRziUGqtT5epSYoI=
+github.com/hashicorp/terraform-plugin-framework-timeouts v0.7.0/go.mod h1:5jm2XK8uqrdiSRfD5O47OoxyGMCnwTcl8eoiDgSa+tc=
 github.com/hashicorp/terraform-plugin-framework-validators v0.19.0 h1:Zz3iGgzxe/1XBkooZCewS0nJAaCFPFPHdNJd8FgE4Ow=
 github.com/hashicorp/terraform-plugin-framework-validators v0.19.0/go.mod h1:GBKTNGbGVJohU03dZ7U8wHqc2zYnMUawgCN+gC0itLc=
 github.com/hashicorp/terraform-plugin-go v0.29.0 h1:1nXKl/nSpaYIUBU1IG/EsDOX0vv+9JxAltQyDMpq5mU=

internal/dbops/interface.go

@@ -2,6 +2,7 @@ package dbops
 
 import (
 	"context"
+	"time"
 )
 
 type Client interface {
@@ -39,7 +40,7 @@ type Client interface {
 	AssociateSettingsProfile(ctx context.Context, id string, roleId *string, userId *string, clusterName *string) error
 	DisassociateSettingsProfile(ctx context.Context, id string, roleId *string, userId *string, clusterName *string) error
 
-	CreateSetting(ctx context.Context, settingsProfileID string, setting Setting, clusterName *string) (*Setting, error)
+	CreateSetting(ctx context.Context, settingsProfileID string, setting Setting, clusterName *string, timeout time.Duration) (*Setting, error)
 	GetSetting(ctx context.Context, settingsProfileID string, name string, clusterName *string) (*Setting, error)
 	DeleteSetting(ctx context.Context, settingsProfileID string, name string, clusterName *string) error
 

internal/dbops/settings.go

@@ -3,7 +3,9 @@ package dbops
 import (
 	"context"
 	"fmt"
+	"time"
 
+	"github.com/hashicorp/terraform-plugin-log/tflog"
 	"github.com/pingcap/errors"
 
 	"github.com/ClickHouse/terraform-provider-clickhousedbops/internal/clickhouseclient"
@@ -18,7 +20,7 @@ type Setting struct {
 	Writability *string
 }
 
-func (i *impl) CreateSetting(ctx context.Context, settingsProfileID string, setting Setting, clusterName *string) (*Setting, error) {
+func (i *impl) CreateSetting(ctx context.Context, settingsProfileID string, setting Setting, clusterName *string, timeout time.Duration) (*Setting, error) {
 	settingsProfile, err := i.GetSettingsProfile(ctx, settingsProfileID, clusterName)
 	if err != nil {
 		return nil, errors.WithMessage(err, "error getting settings profile")
@@ -41,7 +43,40 @@ func (i *impl) CreateSetting(ctx context.Context, settingsProfileID string, sett
 		return nil, errors.WithMessage(err, "error running query")
 	}
 
-	return i.GetSetting(ctx, settingsProfileID, setting.Name, clusterName)
+	// Retry to handle potential replication lag
+	retryCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	backoff := 50 * time.Millisecond
+	for {
+		createdSetting, err := i.GetSetting(retryCtx, settingsProfileID, setting.Name, clusterName)
+		if err != nil {
+			return nil, errors.WithMessage(err, "error retrieving created setting")
+		}
+
+		if createdSetting != nil {
+			return createdSetting, nil
+		}
+
+		tflog.Debug(ctx, "Setting not found, retrying with exponential backoff", map[string]any{
+			"setting_name": setting.Name,
+			"backoff":      backoff.String(),
+		})
+
+		// Context-aware sleep with exponential backoff
+		timer := time.NewTimer(backoff)
+		defer timer.Stop()
+
+		select {
+		case <-retryCtx.Done():
+			return nil, fmt.Errorf(
+				"setting %q was created but could not be retrieved within timeout (%v): %w",
+				setting.Name, timeout, retryCtx.Err(),
+			)
+		case <-timer.C:
+			backoff *= 2
+		}
+	}
 }
 
 func (i *impl) GetSetting(ctx context.Context, settingsProfileID string, name string, clusterName *string) (*Setting, error) {

pkg/resource/setting/model.go

@@ -1,15 +1,17 @@
 package setting
 
 import (
+	"github.com/hashicorp/terraform-plugin-framework-timeouts/resource/timeouts"
 	"github.com/hashicorp/terraform-plugin-framework/types"
 )
 
 type Setting struct {
-	ClusterName       types.String `tfsdk:"cluster_name"`
-	SettingsProfileID types.String `tfsdk:"settings_profile_id"`
-	Name              types.String `tfsdk:"name"`
-	Value             types.String `tfsdk:"value"`
-	Min               types.String `tfsdk:"min"`
-	Max               types.String `tfsdk:"max"`
-	Writability       types.String `tfsdk:"writability"`
+	ClusterName       types.String   `tfsdk:"cluster_name"`
+	SettingsProfileID types.String   `tfsdk:"settings_profile_id"`
+	Name              types.String   `tfsdk:"name"`
+	Value             types.String   `tfsdk:"value"`
+	Min               types.String   `tfsdk:"min"`
+	Max               types.String   `tfsdk:"max"`
+	Writability       types.String   `tfsdk:"writability"`
+	Timeouts          timeouts.Value `tfsdk:"timeouts"`
 }

pkg/resource/setting/setting.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	_ "embed"
 	"fmt"
+	"time"
 
+	"github.com/hashicorp/terraform-plugin-framework-timeouts/resource/timeouts"
 	"github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator"
 	"github.com/hashicorp/terraform-plugin-framework/path"
 	"github.com/hashicorp/terraform-plugin-framework/resource"
@@ -38,7 +40,7 @@ func (r *Resource) Metadata(_ context.Context, req resource.MetadataRequest, res
 	resp.TypeName = req.ProviderTypeName + "_setting"
 }
 
-func (r *Resource) Schema(_ context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) {
+func (r *Resource) Schema(ctx context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) {
 	resp.Schema = schema.Schema{
 		Attributes: map[string]schema.Attribute{
 			"cluster_name": schema.StringAttribute{
@@ -116,6 +118,9 @@ func (r *Resource) Schema(_ context.Context, _ resource.SchemaRequest, resp *res
 					),
 				},
 			},
+			"timeouts": timeouts.Attributes(ctx, timeouts.Opts{
+				Create: true,
+			}),
 		},
 		MarkdownDescription: settingResourceDescription,
 	}
@@ -180,7 +185,14 @@ func (r *Resource) Create(ctx context.Context, req resource.CreateRequest, resp
 		Writability: plan.Writability.ValueStringPointer(),
 	}
 
-	createdSetting, err := r.client.CreateSetting(ctx, plan.SettingsProfileID.ValueString(), setting, plan.ClusterName.ValueStringPointer())
+	createTimeout, diags := plan.Timeouts.Create(ctx, 1*time.Minute)
+	resp.Diagnostics.Append(diags...)
+
+	if resp.Diagnostics.HasError() {
+		return
+	}
+
+	createdSetting, err := r.client.CreateSetting(ctx, plan.SettingsProfileID.ValueString(), setting, plan.ClusterName.ValueStringPointer(), createTimeout)
 	if err != nil {
 		resp.Diagnostics.AddError(
 			"Error Creating ClickHouse Setting",
@@ -189,9 +201,18 @@ func (r *Resource) Create(ctx context.Context, req resource.CreateRequest, resp
 		return
 	}
 
+	if createdSetting == nil {
+		resp.Diagnostics.AddError(
+			"Error Creating ClickHouse Setting",
+			"Created setting returned nil without error",
+		)
+		return
+	}
+
 	state := Setting{
 		ClusterName:       plan.ClusterName,
 		SettingsProfileID: plan.SettingsProfileID,
+		Timeouts:          plan.Timeouts,
 	}
 
 	modelFromApiResponse(&state, *createdSetting)