Retry on timeout, when server doesn't respond (#1161)

* Retry on timeout, when server doesn't respond

Signed-off-by: Romain Philibert <[email protected]>

* update comment

Signed-off-by: Romain <[email protected]>

---------

Signed-off-by: Romain Philibert <[email protected]>
Signed-off-by: Romain <[email protected]>

Author: Filirom1

keycloak/keycloak_client.go

@@ -499,6 +499,30 @@ func (keycloakClient *KeycloakClient) marshal(body interface{}) ([]byte, error)
 	return json.Marshal(body)
 }
 
+func RetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	// do retry on context.Canceled or context.DeadlineExceeded
+	if ctx.Err() != nil {
+		return true, ctx.Err()
+	}
+
+	// 429 Too Many Requests is recoverable. Sometimes the server puts
+	// a Retry-After response header to indicate when the server is
+	// available to start processing request from client.
+	if resp.StatusCode == http.StatusTooManyRequests {
+		return true, nil
+	}
+
+	// Check the response code. We retry on 500-range responses to allow
+	// the server time to recover, as 500's are typically not permanent
+	// errors and may relate to outages on the server side. This will catch
+	// invalid response codes as well, like 0 and 999.
+	if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != http.StatusNotImplemented) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func newHttpClient(tlsInsecureSkipVerify bool, clientTimeout int, caCert string) (*http.Client, error) {
 	cookieJar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: publicsuffix.List,
@@ -519,9 +543,10 @@ func newHttpClient(tlsInsecureSkipVerify bool, clientTimeout int, caCert string)
 	}
 
 	retryClient := retryablehttp.NewClient()
-	retryClient.RetryMax = 1
+	retryClient.CheckRetry = RetryPolicy
+	retryClient.RetryMax = 5
 	retryClient.RetryWaitMin = time.Second * 1
-	retryClient.RetryWaitMax = time.Second * 3
+	retryClient.RetryWaitMax = time.Second * 60
 
 	httpClient := retryClient.StandardClient()
 	httpClient.Timeout = time.Second * time.Duration(clientTimeout)