Skip to content

Commit 2cce42c

Browse files
l1b0kl1b0k
authored
Merge pull request #355 from l1b0k/release-1.2
feat: eip retry on conflict
2 parents 290bf62 + 2669e2f commit 2cce42c

File tree

2 files changed

+81
-55
lines changed

2 files changed

+81
-55
lines changed

‎pkg/aliyun/client/errors/errors.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ const (
3636

3737
// ErrIPNotInCbwp for eip
3838
ErrIPNotInCbwp = "OperationUnsupported.IpNotInCbwp"
39+
40+
// ErrTaskConflict for eip
41+
ErrTaskConflict = "TaskConflict"
3942
)
4043

4144
// define well known err

‎pkg/aliyun/client/vpc.go‎

Lines changed: 78 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ import (
66
"time"
77

88
apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors"
9+
"github.com/AliyunContainerService/terway/pkg/backoff"
910
"github.com/AliyunContainerService/terway/pkg/metric"
1011
"github.com/AliyunContainerService/terway/types"
1112

1213
"github.com/aliyun/alibaba-cloud-sdk-go/services/vpc"
14+
"k8s.io/client-go/util/retry"
1315
)
1416

1517
// DescribeVSwitchByID get vsw by id
@@ -77,17 +79,22 @@ func (a *OpenAPI) AssociateEIPAddress(eipID, eniID, privateIP string) error {
7779
LogFieldENIID: eniID,
7880
})
7981
start := time.Now()
80-
resp, err := a.ClientSet.VPC().AssociateEipAddress(req)
81-
metric.OpenAPILatency.WithLabelValues("AssociateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
82-
if err != nil {
83-
l.WithFields(map[string]interface{}{
84-
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("associate EIP to %s failed, %s", privateIP, err.Error())
8582

86-
return fmt.Errorf("error associate EIP %s, %w", eipID, err)
87-
}
88-
l.WithFields(map[string]interface{}{
89-
LogFieldRequestID: resp.RequestId}).Infof("associate EIP to %s", privateIP)
90-
return nil
83+
return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool {
84+
return apiErr.ErrAssert(apiErr.ErrTaskConflict, err)
85+
}, func() error {
86+
resp, err := a.ClientSet.VPC().AssociateEipAddress(req)
87+
metric.OpenAPILatency.WithLabelValues("AssociateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
88+
if err != nil {
89+
l.WithFields(map[string]interface{}{
90+
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("associate EIP to %s failed, %s", privateIP, err.Error())
91+
92+
return err
93+
}
94+
l.WithFields(map[string]interface{}{
95+
LogFieldRequestID: resp.RequestId}).Infof("associate EIP to %s", privateIP)
96+
return nil
97+
})
9198
}
9299

93100
// UnAssociateEIPAddress un-bind eip
@@ -102,21 +109,26 @@ func (a *OpenAPI) UnAssociateEIPAddress(eipID, eniID, eniIP string) error {
102109
LogFieldAPI: "UnassociateEipAddress",
103110
LogFieldEIPID: eipID,
104111
})
105-
start := time.Now()
106-
resp, err := a.ClientSet.VPC().UnassociateEipAddress(req)
107-
metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
108-
if err != nil {
109-
l.WithFields(map[string]interface{}{
110-
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("unassociate EIP failed, %s", err.Error())
111-
if apiErr.ErrAssert(apiErr.ErrInvalidAllocationIDNotFound, err) ||
112-
apiErr.ErrAssert(apiErr.ErrIncorrectEIPStatus, err) {
113-
return nil
112+
113+
return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool {
114+
return apiErr.ErrAssert(apiErr.ErrTaskConflict, err)
115+
}, func() error {
116+
start := time.Now()
117+
resp, err := a.ClientSet.VPC().UnassociateEipAddress(req)
118+
metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
119+
if err != nil {
120+
l.WithFields(map[string]interface{}{
121+
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("unassociate EIP failed, %s", err.Error())
122+
if apiErr.ErrAssert(apiErr.ErrInvalidAllocationIDNotFound, err) ||
123+
apiErr.ErrAssert(apiErr.ErrIncorrectEIPStatus, err) {
124+
return nil
125+
}
126+
return fmt.Errorf("error unassociate EIP %s, %w", eipID, err)
114127
}
115-
return fmt.Errorf("error unassociate EIP %s, %w", eipID, err)
116-
}
117-
l.WithFields(map[string]interface{}{
118-
LogFieldRequestID: resp.RequestId}).Info("unassociate EIP")
119-
return nil
128+
l.WithFields(map[string]interface{}{
129+
LogFieldRequestID: resp.RequestId}).Info("unassociate EIP")
130+
return nil
131+
})
120132
}
121133

122134
// ReleaseEIPAddress delete EIP
@@ -129,17 +141,21 @@ func (a *OpenAPI) ReleaseEIPAddress(eipID string) error {
129141
LogFieldEIPID: eipID,
130142
})
131143

132-
start := time.Now()
133-
resp, err := a.ClientSet.VPC().ReleaseEipAddress(req)
134-
metric.OpenAPILatency.WithLabelValues("ReleaseEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
135-
if err != nil {
144+
return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool {
145+
return apiErr.ErrAssert(apiErr.ErrTaskConflict, err)
146+
}, func() error {
147+
start := time.Now()
148+
resp, err := a.ClientSet.VPC().ReleaseEipAddress(req)
149+
metric.OpenAPILatency.WithLabelValues("ReleaseEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
150+
if err != nil {
151+
l.WithFields(map[string]interface{}{
152+
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("release EIP failed, %s", err.Error())
153+
return err
154+
}
136155
l.WithFields(map[string]interface{}{
137-
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("release EIP failed, %s", err.Error())
138-
return fmt.Errorf("error release EIP %s, %w", eipID, err)
139-
}
140-
l.WithFields(map[string]interface{}{
141-
LogFieldRequestID: resp.RequestId}).Info("release EIP")
142-
return nil
156+
LogFieldRequestID: resp.RequestId}).Info("release EIP")
157+
return nil
158+
})
143159
}
144160

145161
// AddCommonBandwidthPackageIP add EIP to bandwidth package
@@ -152,18 +168,21 @@ func (a *OpenAPI) AddCommonBandwidthPackageIP(eipID, packageID string) error {
152168
LogFieldAPI: "AddCommonBandwidthPackageIp",
153169
LogFieldEIPID: eipID,
154170
})
155-
156-
start := time.Now()
157-
resp, err := a.ClientSet.VPC().AddCommonBandwidthPackageIp(req)
158-
metric.OpenAPILatency.WithLabelValues("AddCommonBandwidthPackageIp", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
159-
if err != nil {
171+
return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool {
172+
return apiErr.ErrAssert(apiErr.ErrTaskConflict, err)
173+
}, func() error {
174+
start := time.Now()
175+
resp, err := a.ClientSet.VPC().AddCommonBandwidthPackageIp(req)
176+
metric.OpenAPILatency.WithLabelValues("AddCommonBandwidthPackageIp", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
177+
if err != nil {
178+
l.WithFields(map[string]interface{}{
179+
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("add eip failed, %s", err.Error())
180+
return err
181+
}
160182
l.WithFields(map[string]interface{}{
161-
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("add eip failed, %s", err.Error())
162-
return fmt.Errorf("error release EIP %s, %w", eipID, err)
163-
}
164-
l.WithFields(map[string]interface{}{
165-
LogFieldRequestID: resp.RequestId}).Info("add eip success")
166-
return nil
183+
LogFieldRequestID: resp.RequestId}).Info("add eip success")
184+
return nil
185+
})
167186
}
168187

169188
// RemoveCommonBandwidthPackageIP remove EIP from bandwidth package
@@ -177,15 +196,19 @@ func (a *OpenAPI) RemoveCommonBandwidthPackageIP(eipID, packageID string) error
177196
LogFieldEIPID: eipID,
178197
})
179198

180-
start := time.Now()
181-
resp, err := a.ClientSet.VPC().RemoveCommonBandwidthPackageIp(req)
182-
metric.OpenAPILatency.WithLabelValues("RemoveCommonBandwidthPackageIp", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
183-
if err != nil {
199+
return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool {
200+
return apiErr.ErrAssert(apiErr.ErrTaskConflict, err)
201+
}, func() error {
202+
start := time.Now()
203+
resp, err := a.ClientSet.VPC().RemoveCommonBandwidthPackageIp(req)
204+
metric.OpenAPILatency.WithLabelValues("RemoveCommonBandwidthPackageIp", fmt.Sprint(err != nil)).Observe(metric.MsSince(start))
205+
if err != nil {
206+
l.WithFields(map[string]interface{}{
207+
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("remove eip failed, %s", err.Error())
208+
return err
209+
}
184210
l.WithFields(map[string]interface{}{
185-
LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("remove eip failed, %s", err.Error())
186-
return fmt.Errorf("error release EIP %s, %w", eipID, err)
187-
}
188-
l.WithFields(map[string]interface{}{
189-
LogFieldRequestID: resp.RequestId}).Info("remove eip success")
190-
return nil
211+
LogFieldRequestID: resp.RequestId}).Info("remove eip success")
212+
return nil
213+
})
191214
}

0 commit comments

Comments
 (0)