Skip to content

Commit 37f229e

Browse files
s-urbaniaks-urbaniak
authored
CLOUDP-251510: test/helper/retry: add helper for get+mutate+update retries (#1625)
* test/helper/retry: add helper for get+mutate+update retries * address import name
1 parent 81988e5 commit 37f229e

File tree

3 files changed

+156
-28
lines changed

3 files changed

+156
-28
lines changed

‎test/helper/retry/retry.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package retry
2+
3+
import (
4+
"context"
5+
6+
"k8s.io/client-go/util/retry"
7+
"sigs.k8s.io/controller-runtime/pkg/client"
8+
)
9+
10+
// RetryUpdateOnConflict is a wrapper around client-go/util/retry.RetryOnConflict,
11+
// adding the following often repeated actions:
12+
//
13+
// 1. client.Get a resource for the given key
14+
// 2. mutate the retrieved object using the given mutator function.
15+
// 3. client.Update the updated resource and retry on conflict
16+
// using the client-go/util/retry.DefaultRetry strategy.
17+
func RetryUpdateOnConflict[T any](ctx context.Context, k8s client.Client, key client.ObjectKey, mutator func(*T)) (*T, error) {
18+
var obj T
19+
clientObj := any(&obj).(client.Object)
20+
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
21+
if err := k8s.Get(ctx, key, clientObj); err != nil {
22+
return err
23+
}
24+
mutator(&obj)
25+
return k8s.Update(ctx, clientObj)
26+
})
27+
return &obj, err
28+
}

‎test/helper/retry/retry_test.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package retry
2+
3+
import (
4+
"context"
5+
"errors"
6+
"reflect"
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
apierrors "k8s.io/apimachinery/pkg/api/errors"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
"k8s.io/apimachinery/pkg/types"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
16+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
17+
18+
akov2 "github.com/mongodb/mongodb-atlas-kubernetes/v2/pkg/api/v1"
19+
)
20+
21+
func TestRetryUpdateOnConflict(t *testing.T) {
22+
for _, tc := range []struct {
23+
name string
24+
key client.ObjectKey
25+
objects []client.Object
26+
interceptorFuncs interceptor.Funcs
27+
28+
want *akov2.AtlasProject
29+
wantErr string
30+
}{
31+
{
32+
name: "fail immediately if not found",
33+
key: types.NamespacedName{
34+
Name: "foo",
35+
Namespace: "bar",
36+
},
37+
want: &akov2.AtlasProject{},
38+
wantErr: "atlasprojects.atlas.mongodb.com \"foo\" not found",
39+
},
40+
{
41+
name: "succeed if found",
42+
key: types.NamespacedName{
43+
Name: "foo",
44+
Namespace: "bar",
45+
},
46+
objects: []client.Object{
47+
&akov2.AtlasProject{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}},
48+
},
49+
want: &akov2.AtlasProject{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}},
50+
},
51+
{
52+
name: "exhaust on conflict",
53+
key: types.NamespacedName{
54+
Name: "foo",
55+
Namespace: "bar",
56+
},
57+
objects: []client.Object{
58+
&akov2.AtlasProject{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}},
59+
},
60+
interceptorFuncs: interceptor.Funcs{
61+
Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error {
62+
return &apierrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonConflict, Message: "conflict"}}
63+
},
64+
},
65+
want: &akov2.AtlasProject{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}},
66+
wantErr: "conflict",
67+
},
68+
{
69+
name: "fail on any other update error",
70+
key: types.NamespacedName{
71+
Name: "foo",
72+
Namespace: "bar",
73+
},
74+
objects: []client.Object{
75+
&akov2.AtlasProject{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}},
76+
},
77+
interceptorFuncs: interceptor.Funcs{
78+
Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error {
79+
return errors.New("boom")
80+
},
81+
},
82+
want: &akov2.AtlasProject{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}},
83+
wantErr: "boom",
84+
},
85+
} {
86+
t.Run(tc.name, func(t *testing.T) {
87+
testScheme := runtime.NewScheme()
88+
assert.NoError(t, akov2.AddToScheme(testScheme))
89+
k8sClient := fake.NewClientBuilder().
90+
WithScheme(testScheme).
91+
WithObjects(tc.objects...).
92+
WithInterceptorFuncs(tc.interceptorFuncs).
93+
Build()
94+
95+
got, err := RetryUpdateOnConflict(context.Background(), k8sClient, tc.key, func(*akov2.AtlasProject) {})
96+
gotErr := ""
97+
if err != nil {
98+
gotErr = err.Error()
99+
}
100+
101+
if gotErr != tc.wantErr {
102+
t.Errorf("want error %q, got %q", tc.wantErr, gotErr)
103+
}
104+
105+
// ignore unnecessary fields
106+
got.ResourceVersion = ""
107+
108+
if !reflect.DeepEqual(got, tc.want) {
109+
t.Errorf("want AtlasProject %+v, got %+v", tc.want, got)
110+
}
111+
})
112+
}
113+
}

‎test/int/deployment_test.go‎

Lines changed: 15 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import (
1818
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1919
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2020
"k8s.io/apimachinery/pkg/types"
21-
"k8s.io/client-go/util/retry"
2221
"sigs.k8s.io/controller-runtime/pkg/client"
2322

2423
"github.com/mongodb/mongodb-atlas-kubernetes/v2/internal/compat"
@@ -36,6 +35,7 @@ import (
3635
"github.com/mongodb/mongodb-atlas-kubernetes/v2/test/helper/atlas"
3736
"github.com/mongodb/mongodb-atlas-kubernetes/v2/test/helper/conditions"
3837
"github.com/mongodb/mongodb-atlas-kubernetes/v2/test/helper/resources"
38+
akoretry "github.com/mongodb/mongodb-atlas-kubernetes/v2/test/helper/retry"
3939
)
4040

4141
const (
@@ -293,7 +293,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
293293
})
294294

295295
By("Filling token secret with invalid data", func() {
296-
_, err := retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(connectionSecret), func(secret *corev1.Secret) {
296+
_, err := akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(connectionSecret), func(secret *corev1.Secret) {
297297
secret.StringData = map[string]string{
298298
OrgID: "fake", PrivateAPIKey: "fake", PublicAPIKey: "fake",
299299
}
@@ -313,7 +313,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
313313
})
314314

315315
By("Fix the token secret", func() {
316-
_, err := retryUpdateOnConflict(ctx, k8sClient, types.NamespacedName{Namespace: namespace.Name, Name: ConnectionSecretName}, func(secret *corev1.Secret) {
316+
_, err := akoretry.RetryUpdateOnConflict(ctx, k8sClient, types.NamespacedName{Namespace: namespace.Name, Name: ConnectionSecretName}, func(secret *corev1.Secret) {
317317
secret.StringData = secretData()
318318
})
319319
Expect(err).To(BeNil())
@@ -764,7 +764,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
764764
})
765765

766766
By("Updating the Deployment tags with a duplicate key and removing all tags", func() {
767-
_, err := retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
767+
_, err := akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
768768
deployment.Spec.DeploymentSpec.Tags = []*akov2.TagSpec{{Key: "test-1", Value: "value-1"}, {Key: "test-1", Value: "value-2"}}
769769
})
770770
Expect(err).To(BeNil())
@@ -812,7 +812,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
812812
})
813813

814814
By("Updating the Deployment configuration while paused (should fail)", func() {
815-
_, err := retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
815+
_, err := akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
816816
deployment.Spec.DeploymentSpec.BackupEnabled = pointer.MakePtr(false)
817817
})
818818
Expect(err).To(BeNil())
@@ -854,7 +854,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
854854
oldSizeName string
855855
err error
856856
)
857-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
857+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
858858
oldSizeName = deployment.Spec.DeploymentSpec.ReplicationSpecs[0].RegionConfigs[0].ElectableSpecs.InstanceSize
859859
deployment.Spec.DeploymentSpec.ReplicationSpecs[0].RegionConfigs[0].ElectableSpecs = &akov2.Specs{
860860
InstanceSize: "M42",
@@ -1007,7 +1007,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
10071007
performCreate(createdDeployment, 30*time.Minute)
10081008

10091009
var err error
1010-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1010+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
10111011
deployment.ObjectMeta.Annotations = map[string]string{customresource.ReconciliationPolicyAnnotation: customresource.ReconciliationPolicySkip}
10121012
deployment.Spec.DeploymentSpec.Labels = append(createdDeployment.Spec.DeploymentSpec.Labels, common.LabelSpec{
10131013
Key: "some-key",
@@ -1045,7 +1045,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
10451045

10461046
By(fmt.Sprintf("Updating the InstanceSize of Advanced Deployment %s", kube.ObjectKeyFromObject(createdDeployment)), func() {
10471047
var err error
1048-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1048+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
10491049
deployment.Spec.DeploymentSpec.ReplicationSpecs[0].RegionConfigs[0].ElectableSpecs = &akov2.Specs{
10501050
InstanceSize: "M20",
10511051
NodeCount: pointer.MakePtr(3),
@@ -1071,7 +1071,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
10711071

10721072
By(fmt.Sprintf("Enable AutoScaling for the Advanced Deployment %s", kube.ObjectKeyFromObject(createdDeployment)), func() {
10731073
var err error
1074-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1074+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
10751075
regionConfig := deployment.Spec.DeploymentSpec.ReplicationSpecs[0].RegionConfigs[0]
10761076
regionConfig.ElectableSpecs.InstanceSize = "M10"
10771077
regionConfig.ReadOnlySpecs.InstanceSize = "M10"
@@ -1100,7 +1100,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
11001100

11011101
By(fmt.Sprintf("Update Instance Size Margins with AutoScaling for Deployment %s", kube.ObjectKeyFromObject(createdDeployment)), func() {
11021102
var err error
1103-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1103+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
11041104
regionConfig := deployment.Spec.DeploymentSpec.ReplicationSpecs[0].RegionConfigs[0]
11051105
regionConfig.AutoScaling.Compute.MinInstanceSize = "M20"
11061106
regionConfig.ElectableSpecs.InstanceSize = "M20"
@@ -1179,7 +1179,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
11791179
err := compat.JSONCopy(&previousDeployment, createdDeployment)
11801180
Expect(err).NotTo(HaveOccurred())
11811181

1182-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1182+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
11831183
deployment.Spec.DeploymentSpec.ReplicationSpecs[0].
11841184
RegionConfigs[0].
11851185
AutoScaling.
@@ -1206,7 +1206,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
12061206
err := compat.JSONCopy(&previousDeployment, createdDeployment)
12071207
Expect(err).NotTo(HaveOccurred())
12081208

1209-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1209+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
12101210
deployment.Spec.DeploymentSpec.ReplicationSpecs[0].
12111211
RegionConfigs[0].
12121212
ElectableSpecs.InstanceSize = "M20"
@@ -1315,7 +1315,7 @@ var _ = Describe("AtlasDeployment", Label("int", "AtlasDeployment", "deployment-
13151315
//nolint:dupl
13161316
By("Updating the Instance tags with a duplicate key and removing all tags", func() {
13171317
var err error
1318-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1318+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
13191319
deployment.Spec.ServerlessSpec.Tags = []*akov2.TagSpec{{Key: "test-1", Value: "value-1"}, {Key: "test-1", Value: "value-2"}}
13201320
})
13211321
Expect(err).To(BeNil())
@@ -1531,7 +1531,7 @@ var _ = Describe("AtlasDeployment", Ordered, Label("int", "AtlasDeployment", "de
15311531

15321532
Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(createdDeployment), createdDeployment)).Should(Succeed())
15331533
var err error
1534-
createdDeployment, err = retryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
1534+
createdDeployment, err = akoretry.RetryUpdateOnConflict(ctx, k8sClient, client.ObjectKeyFromObject(createdDeployment), func(deployment *akov2.AtlasDeployment) {
15351535
deployment.Spec.BackupScheduleRef = common.ResourceRefNamespaced{
15361536
Name: bScheduleName,
15371537
Namespace: namespace.Name,
@@ -1865,7 +1865,7 @@ func mergedAdvancedDeployment(
18651865
}
18661866

18671867
func performUpdate[T any](ctx context.Context, timeout time.Duration, key client.ObjectKey, mutator func(*T)) *T {
1868-
obj, err := retryUpdateOnConflict(ctx, k8sClient, key, mutator)
1868+
obj, err := akoretry.RetryUpdateOnConflict(ctx, k8sClient, key, mutator)
18691869
Expect(err).To(BeNil())
18701870

18711871
clientObj := any(obj).(api.AtlasCustomResource)
@@ -1875,16 +1875,3 @@ func performUpdate[T any](ctx context.Context, timeout time.Duration, key client
18751875

18761876
return obj
18771877
}
1878-
1879-
func retryUpdateOnConflict[T any](ctx context.Context, k8s client.Client, key client.ObjectKey, mutator func(*T)) (*T, error) {
1880-
var obj T
1881-
clientObj := any(&obj).(client.Object)
1882-
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
1883-
if err := k8s.Get(ctx, key, clientObj); err != nil {
1884-
return err
1885-
}
1886-
mutator(&obj)
1887-
return k8s.Update(ctx, clientObj)
1888-
})
1889-
return &obj, err
1890-
}

0 commit comments

Comments
 (0)