Fix flaky service account tests (#1304)

Sometimes, the SA tests fail with the "service account already exists" error
The issue is due to the fact that Grafana sometimes 500s on SA POSTs, but the SA is actually created, so on the retry, there's a conflict
This PR changes the retry process so that if we 500, we try to find the SA and "import" it before retrying

Author: julienduchesne

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/go-openapi/strfmt v0.22.0
 	github.com/grafana/amixr-api-go-client v0.0.11
 	github.com/grafana/grafana-api-golang-client v0.27.0
-	github.com/grafana/grafana-openapi-client-go v0.0.0-20240118162741-b884e1a072bf
+	github.com/grafana/grafana-openapi-client-go v0.0.0-20240126032018-bd23c00af697
 	github.com/grafana/machine-learning-go-client v0.5.0
 	github.com/grafana/slo-openapi-client/go v0.0.0-20240112175006-de02e75b9d73
 	github.com/grafana/synthetic-monitoring-agent v0.19.3

go.sum

@@ -102,8 +102,8 @@ github.com/grafana/amixr-api-go-client v0.0.11 h1:jlE+5t0tRuCtjbpM81j70Dr2J4eCyS
 github.com/grafana/amixr-api-go-client v0.0.11/go.mod h1:N6x26XUrM5zGtK5zL5vNJnAn2JFMxLFPPLTw/6pDkFE=
 github.com/grafana/grafana-api-golang-client v0.27.0 h1:zIwMXcbCB4n588i3O2N6HfNcQogCNTd/vPkEXTr7zX8=
 github.com/grafana/grafana-api-golang-client v0.27.0/go.mod h1:uNLZEmgKtTjHBtCQMwNn3qsx2mpMb8zU+7T4Xv3NR9Y=
-github.com/grafana/grafana-openapi-client-go v0.0.0-20240118162741-b884e1a072bf h1:bi08tzZ4QEEe4KBDXfJlcO9QwUdvM/ndzrmV3mozzkU=
-github.com/grafana/grafana-openapi-client-go v0.0.0-20240118162741-b884e1a072bf/go.mod h1:af7rlJw/VtbvAfI5VWzYO4p/pT58FXrN6XqZBnkwBxo=
+github.com/grafana/grafana-openapi-client-go v0.0.0-20240126032018-bd23c00af697 h1:8io6OuKJrhH8SEfwXV1ZpAwgaGhJ25/pjaA3vYmMxxE=
+github.com/grafana/grafana-openapi-client-go v0.0.0-20240126032018-bd23c00af697/go.mod h1:af7rlJw/VtbvAfI5VWzYO4p/pT58FXrN6XqZBnkwBxo=
 github.com/grafana/machine-learning-go-client v0.5.0 h1:Q1K+MPSy8vfMm2jsk3WQ7O77cGr2fM5hxwtPSoPc5NU=
 github.com/grafana/machine-learning-go-client v0.5.0/go.mod h1:QFfZz8NkqVF8++skjkKQXJEZfpCYd8S0yTWJUpsLLTA=
 github.com/grafana/slo-openapi-client/go v0.0.0-20240112175006-de02e75b9d73 h1:E5vAeB5q1H3BVeNhtd1dI8RubucJdPwpx/ElNtKD3ls=

internal/resources/grafana/data_source_service_account.go

@@ -2,8 +2,11 @@ package grafana
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/grafana/grafana-openapi-client-go/client"
 	"github.com/grafana/grafana-openapi-client-go/client/service_accounts"
+	"github.com/grafana/grafana-openapi-client-go/models"
 	"github.com/grafana/terraform-provider-grafana/internal/common"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
@@ -30,23 +33,31 @@ func DatasourceServiceAccount() *schema.Resource {
 func DatasourceServiceAccountRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
 	client, orgID := OAPIClientFromNewOrgResource(meta, d)
 	name := d.Get("name").(string)
+	sa, err := findServiceAccountByName(client, name)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+	d.SetId(MakeOrgResourceID(orgID, sa.ID))
+	return ReadServiceAccount(ctx, d, meta)
+}
+
+func findServiceAccountByName(client *client.GrafanaHTTPAPI, name string) (*models.ServiceAccountDTO, error) {
 	var page int64 = 0
 	for {
 		params := service_accounts.NewSearchOrgServiceAccountsWithPagingParams().WithPage(&page)
 		resp, err := client.ServiceAccounts.SearchOrgServiceAccountsWithPaging(params)
 		if err != nil {
-			return diag.FromErr(err)
+			return nil, err
 		}
 		serviceAccounts := resp.Payload.ServiceAccounts
 		if len(serviceAccounts) == 0 {
 			break
 		}
 		for _, sa := range serviceAccounts {
 			if sa.Name == name {
-				d.SetId(MakeOrgResourceID(orgID, sa.ID))
-				return ReadServiceAccount(ctx, d, meta)
+				return sa, nil
 			}
 		}
 	}
-	return diag.Errorf("service account %q not found", name)
+	return nil, fmt.Errorf("service account %q not found", name)
 }

internal/resources/grafana/resource_service_account.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/grafana/grafana-openapi-client-go/client/service_accounts"
 	"github.com/grafana/grafana-openapi-client-go/models"
 	"github.com/grafana/terraform-provider-grafana/internal/common"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 )
@@ -60,18 +62,38 @@ func CreateServiceAccount(ctx context.Context, d *schema.ResourceData, meta inte
 	defer serviceAccountCreateMutex.Unlock()
 
 	client, orgID := OAPIClientFromNewOrgResource(meta, d)
+	client = client.WithRetries(0, 0) // Disable retries to have our own retry logic
 	req := models.CreateServiceAccountForm{
 		Name:       d.Get("name").(string),
 		Role:       d.Get("role").(string),
 		IsDisabled: d.Get("is_disabled").(bool),
 	}
 
-	params := service_accounts.NewCreateServiceAccountParams().WithBody(&req)
-	resp, err := client.ServiceAccounts.CreateServiceAccount(params)
+	var sa *models.ServiceAccountDTO
+	err := retry.RetryContext(ctx, 10*time.Second, func() *retry.RetryError {
+		params := service_accounts.NewCreateServiceAccountParams().WithBody(&req)
+		resp, err := client.ServiceAccounts.CreateServiceAccount(params)
+		if err == nil {
+			sa = resp.Payload
+			return nil
+		}
+
+		if err, ok := err.(*service_accounts.CreateServiceAccountInternalServerError); ok {
+			// Sometimes on 500s, the service account is created but the response is not returned.
+			// If we just retry, it will conflict because the SA was actually created.
+			foundSa, readErr := findServiceAccountByName(client, req.Name)
+			if readErr != nil {
+				return retry.RetryableError(err)
+			}
+			sa = foundSa
+			return nil
+		}
+		return retry.NonRetryableError(err)
+	})
 	if err != nil {
 		return diag.FromErr(err)
 	}
-	d.SetId(MakeOrgResourceID(orgID, resp.Payload.ID))
+	d.SetId(MakeOrgResourceID(orgID, sa.ID))
 
 	return ReadServiceAccount(ctx, d, meta)
 }