Merge branch 'main' into master

Author: chrismilson

keycloak/keycloak_client.go

@@ -332,6 +332,7 @@ func (keycloakClient *KeycloakClient) sendRequest(ctx context.Context, request *
 	if err != nil {
 		return nil, "", fmt.Errorf("error sending request: %v", err)
 	}
+	defer response.Body.Close()
 
 	// Unauthorized: Token could have expired
 	// Forbidden: After creating a realm, following GETs for the realm return 403 until you refresh
@@ -354,10 +355,9 @@ func (keycloakClient *KeycloakClient) sendRequest(ctx context.Context, request *
 		if err != nil {
 			return nil, "", fmt.Errorf("error sending request after refresh: %v", err)
 		}
+		defer response.Body.Close()
 	}
 
-	defer response.Body.Close()
-
 	responseBody, err := io.ReadAll(response.Body)
 	if err != nil {
 		return nil, "", err
@@ -499,6 +499,30 @@ func (keycloakClient *KeycloakClient) marshal(body interface{}) ([]byte, error)
 	return json.Marshal(body)
 }
 
+func RetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	// do retry on context.Canceled or context.DeadlineExceeded
+	if ctx.Err() != nil {
+		return true, ctx.Err()
+	}
+
+	// 429 Too Many Requests is recoverable. Sometimes the server puts
+	// a Retry-After response header to indicate when the server is
+	// available to start processing request from client.
+	if resp.StatusCode == http.StatusTooManyRequests {
+		return true, nil
+	}
+
+	// Check the response code. We retry on 500-range responses to allow
+	// the server time to recover, as 500's are typically not permanent
+	// errors and may relate to outages on the server side. This will catch
+	// invalid response codes as well, like 0 and 999.
+	if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != http.StatusNotImplemented) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func newHttpClient(tlsInsecureSkipVerify bool, clientTimeout int, caCert string) (*http.Client, error) {
 	cookieJar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: publicsuffix.List,
@@ -519,9 +543,10 @@ func newHttpClient(tlsInsecureSkipVerify bool, clientTimeout int, caCert string)
 	}
 
 	retryClient := retryablehttp.NewClient()
-	retryClient.RetryMax = 1
+	retryClient.CheckRetry = RetryPolicy
+	retryClient.RetryMax = 5
 	retryClient.RetryWaitMin = time.Second * 1
-	retryClient.RetryWaitMax = time.Second * 3
+	retryClient.RetryWaitMax = time.Second * 60
 
 	httpClient := retryClient.StandardClient()
 	httpClient.Timeout = time.Second * time.Duration(clientTimeout)