fix(iam): handle http 409 transient error

* exponentialy retry Add amd Remove group member requests

Author: Gnoale

internal/services/iam/group_membership.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	iam "github.com/scaleway/scaleway-sdk-go/api/iam/v1alpha1"
 	"github.com/scaleway/scaleway-sdk-go/scw"
 	"github.com/scaleway/terraform-provider-scaleway/v2/internal/httperrors"
+	"github.com/scaleway/terraform-provider-scaleway/v2/internal/transport"
 	"github.com/scaleway/terraform-provider-scaleway/v2/internal/types"
 )
 
@@ -53,11 +55,11 @@ func resourceIamGroupMembershipCreate(ctx context.Context, d *schema.ResourceDat
 	userID := types.ExpandStringPtr(d.Get("user_id"))
 	applicationID := types.ExpandStringPtr(d.Get("application_id"))
 
-	group, err := api.AddGroupMember(&iam.AddGroupMemberRequest{
+	group, err := MakeGroupRequest(ctx, api, &iam.AddGroupMemberRequest{
 		GroupID:       d.Get("group_id").(string),
 		UserID:        userID,
 		ApplicationID: applicationID,
-	}, scw.WithContext(ctx))
+	})
 	if err != nil {
 		return diag.FromErr(err)
 	}
@@ -74,7 +76,7 @@ func resourceIamGroupMembershipRead(ctx context.Context, d *schema.ResourceData,
 	if err != nil {
 		return diag.FromErr(err)
 	}
-
+	// http GET request should not return a 409 error
 	group, err := api.GetGroup(&iam.GetGroupRequest{
 		GroupID: groupID,
 	}, scw.WithContext(ctx))
@@ -139,7 +141,7 @@ func resourceIamGroupMembershipDelete(ctx context.Context, d *schema.ResourceDat
 		req.ApplicationID = &applicationID
 	}
 
-	_, err = api.RemoveGroupMember(req, scw.WithContext(ctx))
+	_, err = MakeGroupRequest(ctx, api, req)
 	if err != nil {
 		if httperrors.Is404(err) {
 			d.SetId("")
@@ -178,3 +180,59 @@ func ExpandGroupMembershipID(id string) (groupID string, userID string, applicat
 
 	return
 }
+
+func MakeGroupRequest(ctx context.Context, api *iam.API, request any) (*iam.Group, error) {
+	retryInterval := 50 * time.Millisecond
+	maxRetries := 10
+
+	if transport.DefaultWaitRetryInterval != nil {
+		retryInterval = *transport.DefaultWaitRetryInterval
+	}
+
+	switch req := request.(type) {
+	case *iam.AddGroupMemberRequest:
+		for i := range maxRetries {
+			response, err := api.AddGroupMember(req, scw.WithContext(ctx))
+			if err != nil {
+				if handleTransientError(err, req.GroupID, retryInterval, i) {
+					continue
+				}
+
+				return nil, err
+			}
+
+			return response, nil
+		}
+
+		return nil, fmt.Errorf("failed to add group member after %d retries", maxRetries)
+
+	case *iam.RemoveGroupMemberRequest:
+		for i := range maxRetries {
+			response, err := api.RemoveGroupMember(req, scw.WithContext(ctx))
+			if err != nil {
+				if handleTransientError(err, req.GroupID, retryInterval, i) {
+					continue
+				}
+
+				return nil, err
+			}
+
+			return response, nil
+		}
+
+		return nil, fmt.Errorf("failed to remove group member after %d retries", maxRetries)
+
+	default:
+		return nil, fmt.Errorf("invalid request type: %T", req)
+	}
+}
+
+func handleTransientError(err error, groupID string, retryInterval time.Duration, maxRetries int) bool {
+	if httperrors.Is409(err) && strings.Contains(err.Error(), fmt.Sprintf("resource group with ID %s is in a transient state: updating", groupID)) {
+		time.Sleep(retryInterval * time.Duration(maxRetries)) // lintignore: R018
+
+		return true
+	}
+
+	return false
+}