Skip to content

Commit 794b8f9

Browse files
l1b0kl1b0k
committed
feat: allow backoff config
1 parent 556d236 commit 794b8f9

File tree

11 files changed

+128
-93
lines changed

11 files changed

+128
-93
lines changed

‎daemon/daemon.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616

1717
"github.com/AliyunContainerService/terway/pkg/aliyun"
1818
podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
19+
"github.com/AliyunContainerService/terway/pkg/backoff"
1920
terwayIP "github.com/AliyunContainerService/terway/pkg/ip"
2021
"github.com/AliyunContainerService/terway/pkg/ipam"
2122
"github.com/AliyunContainerService/terway/pkg/link"
@@ -1293,6 +1294,8 @@ func newNetworkService(configFilePath, kubeconfig, master, daemonMode string) (r
12931294
return nil, fmt.Errorf("failed parse config: %v", err)
12941295
}
12951296

1297+
backoff.OverrideBackoff(config.BackoffOverride)
1298+
12961299
if len(dynamicCfg) == 0 {
12971300
serviceLog.Infof("got config: %+v from: %+v", config, configFilePath)
12981301
} else {

‎daemon/k8s.go‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/AliyunContainerService/terway/deviceplugin"
1616
podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
17+
"github.com/AliyunContainerService/terway/pkg/backoff"
1718
"github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned/typed/network.alibabacloud.com/v1beta1"
1819
"github.com/AliyunContainerService/terway/pkg/storage"
1920
"github.com/AliyunContainerService/terway/pkg/tracing"
@@ -179,12 +180,7 @@ func (k *k8s) PatchEipInfo(info *types.PodInfo) error {
179180
}
180181

181182
func (k *k8s) WaitPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, err error) {
182-
err = wait.ExponentialBackoff(wait.Backoff{
183-
Duration: time.Second * 5,
184-
Factor: 2,
185-
Jitter: 0.3,
186-
Steps: 3,
187-
}, func() (bool, error) {
183+
err = wait.ExponentialBackoff(backoff.Backoff(backoff.WaitPodENIStatus), func() (bool, error) {
188184
podEni, err = k.podEniClient.PodENIs(info.Namespace).Get(context.TODO(), info.Name, metav1.GetOptions{
189185
ResourceVersion: "0",
190186
})
@@ -205,12 +201,7 @@ func (k *k8s) WaitPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, e
205201
}
206202

207203
func (k *k8s) GetPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, err error) {
208-
err = wait.ExponentialBackoff(wait.Backoff{
209-
Duration: time.Second * 1,
210-
Factor: 2,
211-
Jitter: 0.3,
212-
Steps: 3,
213-
}, func() (bool, error) {
204+
err = wait.ExponentialBackoff(backoff.Backoff(backoff.WaitPodENIStatus), func() (bool, error) {
214205
podEni, err = k.podEniClient.PodENIs(info.Namespace).Get(context.TODO(), info.Name, metav1.GetOptions{
215206
ResourceVersion: "0",
216207
})

‎pkg/aliyun/aliyun.go‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"time"
1010

1111
apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/errors"
12+
"github.com/AliyunContainerService/terway/pkg/backoff"
1213
"github.com/AliyunContainerService/terway/pkg/ip"
1314
"github.com/AliyunContainerService/terway/pkg/ipam"
1415
"github.com/AliyunContainerService/terway/pkg/logger"
@@ -87,7 +88,7 @@ func (e *Impl) AllocateENI(ctx context.Context, vSwitch string, securityGroups [
8788
}()
8889

8990
var innerErr error
90-
err = wait.ExponentialBackoffWithContext(ctx, ENIOpBackoff, func() (bool, error) {
91+
err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.WaitENIStatus), func() (bool, error) {
9192
innerErr = e.AttachNetworkInterface(ctx, resp.NetworkInterfaceId, instanceID, "")
9293
if innerErr != nil {
9394
return false, nil
@@ -105,8 +106,8 @@ func (e *Impl) AllocateENI(ctx context.Context, vSwitch string, securityGroups [
105106

106107
start := time.Now()
107108
// bind status is async api, sleep for first bind status inspect
108-
time.Sleep(eniStateBackoff.Duration)
109-
eniStatus, err := e.WaitForNetworkInterface(ctx, resp.NetworkInterfaceId, ENIStatusInUse, eniStateBackoff, false)
109+
time.Sleep(backoff.Backoff(backoff.WaitENIStatus).Duration)
110+
eniStatus, err := e.WaitForNetworkInterface(ctx, resp.NetworkInterfaceId, ENIStatusInUse, backoff.Backoff(backoff.WaitENIStatus), false)
110111
metric.OpenAPILatency.WithLabelValues("WaitForNetworkInterfaceBind/"+string(ENIStatusInUse), fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
111112

112113
if err != nil {
@@ -115,8 +116,7 @@ func (e *Impl) AllocateENI(ctx context.Context, vSwitch string, securityGroups [
115116

116117
var eni *types.ENI
117118
// backoff get eni config
118-
err = wait.ExponentialBackoffWithContext(ctx,
119-
eniStateBackoff,
119+
err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.WaitENIStatus),
120120
func() (done bool, err error) {
121121
l, ok := GetLimit(GetInstanceMeta().InstanceType)
122122
if !ok {
@@ -147,8 +147,7 @@ func (e *Impl) FreeENI(ctx context.Context, eniID, instanceID string) error {
147147

148148
func (e *Impl) destroyInterface(ctx context.Context, eniID, instanceID, trunkENIID string) error {
149149
var innerErr error
150-
err := wait.ExponentialBackoffWithContext(ctx,
151-
eniReleaseBackoff,
150+
err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIRelease),
152151
func() (done bool, err error) {
153152
innerErr = e.DetachNetworkInterface(ctx, eniID, instanceID, trunkENIID)
154153
if innerErr != nil {
@@ -163,11 +162,10 @@ func (e *Impl) destroyInterface(ctx context.Context, eniID, instanceID, trunkENI
163162
tracing.DisposeResourceFailed, fmtErr)
164163
}
165164

166-
time.Sleep(eniStateBackoff.Duration)
165+
time.Sleep(backoff.Backoff(backoff.WaitENIStatus).Duration)
167166

168167
// backoff delete network interface
169-
err = wait.ExponentialBackoffWithContext(ctx,
170-
eniReleaseBackoff,
168+
err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps),
171169
func() (done bool, err error) {
172170
innerErr = e.DeleteNetworkInterface(context.Background(), eniID)
173171
if innerErr != nil {
@@ -272,7 +270,7 @@ func (e *Impl) AssignNIPsForENI(ctx context.Context, eniID, mac string, count in
272270

273271
if e.ipFamily.IPv4 {
274272
var innerErr error
275-
err = wait.ExponentialBackoffWithContext(ctx, ENIOpBackoff, func() (bool, error) {
273+
err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func() (bool, error) {
276274
ipv4s, innerErr = e.AssignPrivateIPAddress(ctx, eniID, count)
277275
if innerErr != nil {
278276
if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) {
@@ -291,8 +289,7 @@ func (e *Impl) AssignNIPsForENI(ctx context.Context, eniID, mac string, count in
291289
wg.Add(1)
292290
go func() {
293291
defer wg.Done()
294-
v4Err = wait.ExponentialBackoffWithContext(ctx,
295-
MetadataAssignPrivateIPBackoff,
292+
v4Err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.MetaAssignPrivateIP),
296293
func() (done bool, err error) {
297294
remoteIPs, err := e.metadata.GetENIPrivateAddressesByMAC(mac)
298295
if err != nil {
@@ -309,7 +306,7 @@ func (e *Impl) AssignNIPsForENI(ctx context.Context, eniID, mac string, count in
309306

310307
if e.ipFamily.IPv6 {
311308
var innerErr error
312-
err = wait.ExponentialBackoffWithContext(ctx, ENIOpBackoff, func() (bool, error) {
309+
err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func() (bool, error) {
313310
ipv6s, innerErr = e.AssignIpv6Addresses(ctx, eniID, count)
314311
if innerErr != nil {
315312
if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) {
@@ -328,8 +325,7 @@ func (e *Impl) AssignNIPsForENI(ctx context.Context, eniID, mac string, count in
328325
wg.Add(1)
329326
go func() {
330327
defer wg.Done()
331-
v6Err = wait.ExponentialBackoffWithContext(ctx,
332-
MetadataAssignPrivateIPBackoff,
328+
v6Err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.MetaAssignPrivateIP),
333329
func() (done bool, err error) {
334330
remoteIPs, err := e.metadata.GetENIPrivateIPv6AddressesByMAC(mac)
335331
if err != nil {
@@ -365,7 +361,7 @@ func (e *Impl) unAssignIPsForENIUnSafe(ctx context.Context, eniID, mac string, i
365361
if len(ipv4s) > 0 {
366362
var innerErr error
367363

368-
err := wait.ExponentialBackoffWithContext(ctx, ENIOpBackoff, func() (bool, error) {
364+
err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func() (bool, error) {
369365
innerErr = e.UnAssignPrivateIPAddresses(ctx, eniID, ipv4s)
370366
if innerErr != nil {
371367
return false, nil
@@ -380,7 +376,7 @@ func (e *Impl) unAssignIPsForENIUnSafe(ctx context.Context, eniID, mac string, i
380376
if len(ipv6s) > 0 {
381377
var innerErr error
382378

383-
err := wait.ExponentialBackoffWithContext(ctx, ENIOpBackoff, func() (bool, error) {
379+
err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func() (bool, error) {
384380
innerErr = e.UnAssignIpv6Addresses(ctx, eniID, ipv6s)
385381
if innerErr != nil {
386382
return false, nil
@@ -402,12 +398,11 @@ func (e *Impl) unAssignIPsForENIUnSafe(ctx context.Context, eniID, mac string, i
402398
start := time.Now()
403399

404400
// unassignPrivateIpAddresses is async api, sleep for first ip addr inspect
405-
time.Sleep(MetadataUnAssignPrivateIPBackoff.Duration)
401+
time.Sleep(backoff.Backoff(backoff.MetaUnAssignPrivateIP).Duration)
406402
// backoff get interface addresses
407403
var innerErr error
408404

409-
err := wait.ExponentialBackoffWithContext(ctx,
410-
MetadataUnAssignPrivateIPBackoff,
405+
err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.MetaUnAssignPrivateIP),
411406
func() (done bool, err error) {
412407
if len(ipv4s) > 0 {
413408
remoteIPs, err := e.metadata.GetENIPrivateAddressesByMAC(mac)

‎pkg/aliyun/eip.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"time"
99

1010
apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/errors"
11+
"github.com/AliyunContainerService/terway/pkg/backoff"
1112
"github.com/AliyunContainerService/terway/pkg/ip"
1213
"github.com/AliyunContainerService/terway/pkg/metric"
1314
"github.com/AliyunContainerService/terway/types"
@@ -25,7 +26,7 @@ func (e *Impl) AllocateEipAddress(ctx context.Context, bandwidth int, chargeType
2526
err error
2627
)
2728
var eni *ecs.NetworkInterfaceSet
28-
eni, err = e.WaitForNetworkInterface(ctx, eniID, "", ENIOpBackoff, false)
29+
eni, err = e.WaitForNetworkInterface(ctx, eniID, "", backoff.Backoff(backoff.ENIOps), false)
2930
if err != nil {
3031
return nil, err
3132
}
@@ -128,7 +129,7 @@ func (e *Impl) AllocateEipAddress(ctx context.Context, bandwidth int, chargeType
128129
time.Sleep(3 * time.Second)
129130

130131
start := time.Now()
131-
_, err = e.WaitForEIP(eipID, eipStatusAvailable, eniStateBackoff)
132+
_, err = e.WaitForEIP(eipID, eipStatusAvailable, backoff.Backoff(backoff.WaitENIStatus))
132133
metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress/Async", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
133134
if err != nil {
134135
return nil, fmt.Errorf("error wait for eip to status Available: %v", err)
@@ -149,7 +150,7 @@ func (e *Impl) AllocateEipAddress(ctx context.Context, bandwidth int, chargeType
149150
time.Sleep(3 * time.Second)
150151

151152
start := time.Now()
152-
_, err = e.WaitForEIP(eipInfo.ID, "InUse", eniStateBackoff)
153+
_, err = e.WaitForEIP(eipInfo.ID, "InUse", backoff.Backoff(backoff.WaitENIStatus))
153154
metric.OpenAPILatency.WithLabelValues("AssociateEipAddress/Async", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
154155
if err != nil {
155156
return nil, fmt.Errorf("wait for eip error: %v", err)
@@ -163,7 +164,7 @@ func (e *Impl) AllocateEipAddress(ctx context.Context, bandwidth int, chargeType
163164
// 3. if eip is not bind ,return code is IncorrectEipStatus
164165
func (e *Impl) UnassociateEipAddress(ctx context.Context, eipID, eniID, eniIP string) error {
165166
var innerErr error
166-
err := wait.ExponentialBackoff(ENIOpBackoff,
167+
err := wait.ExponentialBackoff(backoff.Backoff(backoff.ENIOps),
167168
func() (done bool, err error) {
168169
// we check eip binding is not changed
169170
var eips []vpc.EipAddress
@@ -200,7 +201,7 @@ func (e *Impl) UnassociateEipAddress(ctx context.Context, eipID, eniID, eniIP st
200201
}
201202

202203
func (e *Impl) ReleaseEipAddress(ctx context.Context, eipID, eniID string, eniIP net.IP) error {
203-
eip, err := e.WaitForEIP(eipID, "", eniStateBackoff)
204+
eip, err := e.WaitForEIP(eipID, "", backoff.Backoff(backoff.WaitENIStatus))
204205
if err != nil {
205206
return fmt.Errorf("error release eip: %w", err)
206207
}
@@ -215,7 +216,7 @@ func (e *Impl) ReleaseEipAddress(ctx context.Context, eipID, eniID string, eniIP
215216
if err == nil {
216217
time.Sleep(3 * time.Second)
217218
start := time.Now()
218-
eip, err = e.WaitForEIP(eipID, eipStatusAvailable, eniStateBackoff)
219+
eip, err = e.WaitForEIP(eipID, eipStatusAvailable, backoff.Backoff(backoff.WaitENIStatus))
219220
metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress/Async", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
220221
if err != nil {
221222
logrus.Errorf("wait timeout UnassociateEipAddress for eni: %v, %v, %v", eniID, eniIP, err)
@@ -226,7 +227,7 @@ func (e *Impl) ReleaseEipAddress(ctx context.Context, eipID, eniID string, eniIP
226227
}
227228
var innerErr error
228229
if eip.Status == eipStatusAvailable {
229-
err = wait.ExponentialBackoff(eniReleaseBackoff, func() (done bool, err error) {
230+
err = wait.ExponentialBackoff(backoff.Backoff(backoff.ENIRelease), func() (done bool, err error) {
230231
innerErr = e.releaseEIPAddress(eip.AllocationId)
231232
if innerErr != nil {
232233
return false, nil

‎pkg/aliyun/instance.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"time"
77

88
"github.com/AliyunContainerService/terway/pkg/aliyun/metadata"
9+
"github.com/AliyunContainerService/terway/pkg/backoff"
910
"github.com/AliyunContainerService/terway/pkg/logger"
1011
"github.com/AliyunContainerService/terway/pkg/metric"
1112
"github.com/AliyunContainerService/terway/pkg/utils"
@@ -134,7 +135,7 @@ func UpdateFromAPI(client *ecs.Client, instanceType string) error {
134135
}
135136
var innerErr error
136137
var resp *ecs.DescribeInstanceTypesResponse
137-
err := wait.ExponentialBackoff(ENIOpBackoff,
138+
err := wait.ExponentialBackoff(backoff.Backoff(backoff.DefaultKey),
138139
func() (done bool, err error) {
139140
start := time.Now()
140141
resp, innerErr = client.DescribeInstanceTypes(req)

‎pkg/aliyun/utils.go‎

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77

88
"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
99
"github.com/aliyun/alibaba-cloud-sdk-go/services/vpc"
10-
"k8s.io/apimachinery/pkg/util/wait"
1110
)
1211

1312
type Client interface {
@@ -43,48 +42,6 @@ const (
4342
ENITypeMember ENIType = "Member"
4443
)
4544

46-
var (
47-
// ENIOpBackoff about 300s backoff
48-
ENIOpBackoff = wait.Backoff{
49-
Duration: time.Second * 5,
50-
Factor: 2,
51-
Jitter: 0.3,
52-
Steps: 6,
53-
}
54-
55-
// eniStateBackoff about 600s backoff
56-
eniStateBackoff = wait.Backoff{
57-
Duration: time.Second * 4,
58-
Factor: 2,
59-
Jitter: 0.3,
60-
Steps: 7,
61-
}
62-
63-
// eniReleaseBackoff about 1200s backoff
64-
eniReleaseBackoff = wait.Backoff{
65-
Duration: time.Second * 4,
66-
Factor: 2,
67-
Jitter: 0.5,
68-
Steps: 8,
69-
}
70-
71-
// MetadataAssignPrivateIPBackoff about 10s backoff
72-
MetadataAssignPrivateIPBackoff = wait.Backoff{
73-
Duration: time.Millisecond * 1100,
74-
Factor: 1,
75-
Jitter: 0.2,
76-
Steps: 10,
77-
}
78-
79-
// MetadataUnAssignPrivateIPBackoff about 10s backoff
80-
MetadataUnAssignPrivateIPBackoff = wait.Backoff{
81-
Duration: time.Millisecond * 1100,
82-
Factor: 1,
83-
Jitter: 0.2,
84-
Steps: 10,
85-
}
86-
)
87-
8845
func generateEniName() string {
8946
b := make([]byte, 3)
9047
rand.Seed(time.Now().UnixNano())

0 commit comments

Comments
 (0)