Skip to content

Commit 90b68fe

Browse files
authored
Add exponential backoff when generating runner reg tokens (#3724)
1 parent 1be410b commit 90b68fe

File tree

2 files changed

+19
-3
lines changed

2 files changed

+19
-3
lines changed

github/actions/client.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"errors"
1111
"fmt"
1212
"io"
13+
"math/rand"
1314
"net/http"
1415
"net/url"
1516
"strconv"
@@ -1139,15 +1140,30 @@ func (c *Client) getActionsServiceAdminConnection(ctx context.Context, rt *regis
11391140
}
11401141

11411142
retry++
1142-
if retry > 3 {
1143+
if retry > 5 {
11431144
return nil, fmt.Errorf("unable to register runner after 3 retries: %w", &GitHubAPIError{
11441145
StatusCode: resp.StatusCode,
11451146
RequestID: resp.Header.Get(HeaderGitHubRequestID),
11461147
Err: innerErr,
11471148
})
11481149
}
1149-
time.Sleep(time.Duration(500 * int(time.Millisecond) * (retry + 1)))
1150+
// Add exponential backoff + jitter to avoid thundering herd
1151+
// This will generate a backoff schedule:
1152+
// 1: 1s
1153+
// 2: 3s
1154+
// 3: 4s
1155+
// 4: 8s
1156+
// 5: 17s
1157+
baseDelay := 500 * time.Millisecond
1158+
jitter := time.Duration(rand.Intn(1000))
1159+
maxDelay := 20 * time.Second
1160+
delay := baseDelay*(1<<retry) + jitter
1161+
1162+
if delay > maxDelay {
1163+
delay = maxDelay
1164+
}
11501165

1166+
time.Sleep(delay)
11511167
}
11521168

11531169
var actionsServiceAdminConnection *ActionsServiceAdminConnection

github/actions/github_api_request_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ func TestNewActionsServiceRequest(t *testing.T) {
170170
}
171171
failures := 0
172172
unauthorizedHandler := func(w http.ResponseWriter, r *http.Request) {
173-
if failures < 2 {
173+
if failures < 5 {
174174
failures++
175175
w.Header().Set("Content-Type", "application/json")
176176
w.WriteHeader(http.StatusUnauthorized)

0 commit comments

Comments
 (0)