Skip to content
This repository was archived by the owner on Dec 12, 2025. It is now read-only.

Commit e82765d

Browse files
authored
CLOUDP-59487: E2E Test to Scale a Replica Set (#28)
1 parent 6b0e1ea commit e82765d

File tree

10 files changed

+167
-109
lines changed

10 files changed

+167
-109
lines changed

.evergreen.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,13 @@ tasks:
132132
vars:
133133
test: replica_set_readiness_probe
134134

135+
- name: e2e_test_replica_set_scale
136+
commands:
137+
- func: clone
138+
- func: setup_kubernetes_environment
139+
- func: run_e2e_test
140+
vars:
141+
test: replica_set_scale
135142

136143
buildvariants:
137144
- name: go_unit_tests
@@ -153,6 +160,7 @@ buildvariants:
153160
tasks:
154161
- name: e2e_test_replica_set
155162
- name: e2e_test_replica_set_readiness_probe
163+
- name: e2e_test_replica_set_scale
156164

157165
- name: init_test_run
158166
display_name: init_test_run

cmd/manager/main.go

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ package main
22

33
import (
44
"fmt"
5+
"os"
6+
57
"github.com/mongodb/mongodb-kubernetes-operator/pkg/apis"
68
"github.com/mongodb/mongodb-kubernetes-operator/pkg/controller"
79
"go.uber.org/zap"
8-
"os"
910
"sigs.k8s.io/controller-runtime/pkg/client/config"
1011
"sigs.k8s.io/controller-runtime/pkg/manager"
1112
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
@@ -25,15 +26,24 @@ func configureLogger() (*zap.Logger, error) {
2526
return logger, err
2627
}
2728

29+
func hasRequiredVariables(logger *zap.Logger, envVariables ...string) bool {
30+
allPresent := true
31+
for _, envVariable := range envVariables {
32+
if _, envSpecified := os.LookupEnv(envVariable); !envSpecified {
33+
logger.Error(fmt.Sprintf("required environment variable %s not found", envVariable))
34+
allPresent = false
35+
}
36+
}
37+
return allPresent
38+
}
39+
2840
func main() {
2941
log, err := configureLogger()
3042
if err != nil {
3143
os.Exit(1)
3244
}
3345

34-
// TODO: implement mechanism to specify required/optional environment variables
35-
if _, agentImageSpecified := os.LookupEnv("AGENT_IMAGE"); !agentImageSpecified {
36-
log.Error("required environment variable AGENT_IMAGE not found")
46+
if !hasRequiredVariables(log, "AGENT_IMAGE") {
3747
os.Exit(1)
3848
}
3949

deploy/operator.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ spec:
1515
serviceAccountName: mongodb-kubernetes-operator
1616
containers:
1717
- name: mongodb-kubernetes-operator
18-
# Replace this with the built image name
1918
image: quay.io/chatton/mongodb-kubernetes-operator
2019
command:
2120
- mongodb-kubernetes-operator

pkg/apis/mongodb/v1/mongodb_types.go

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import (
44
"fmt"
55
"strings"
66

7-
appsv1 "k8s.io/api/apps/v1"
8-
corev1 "k8s.io/api/core/v1"
97
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
108
)
119

@@ -80,43 +78,6 @@ func (m MongoDB) GetFCV() string {
8078
return strings.Join(parts[:minorIndex+1], ".")
8179
}
8280

83-
// TODO: build the correct statefulset - this is a dummy implementation
84-
// BuildStatefulSet constructs an instance of appsv1.StatefulSet
85-
// which should be created during reconciliation
86-
func (m MongoDB) BuildStatefulSet() appsv1.StatefulSet {
87-
labels := map[string]string{
88-
"app": m.ServiceName(),
89-
}
90-
replicas := int32(m.Spec.Members)
91-
return appsv1.StatefulSet{
92-
ObjectMeta: metav1.ObjectMeta{
93-
Name: m.Name,
94-
Namespace: m.Namespace,
95-
},
96-
Spec: appsv1.StatefulSetSpec{
97-
Replicas: &replicas,
98-
Template: corev1.PodTemplateSpec{
99-
ObjectMeta: metav1.ObjectMeta{
100-
Name: m.Name,
101-
Namespace: m.Namespace,
102-
Labels: labels,
103-
},
104-
Spec: corev1.PodSpec{
105-
Containers: []corev1.Container{
106-
{
107-
Name: "mongo",
108-
Image: fmt.Sprintf("mongo:%s", m.Spec.Version),
109-
},
110-
},
111-
},
112-
},
113-
Selector: &metav1.LabelSelector{
114-
MatchLabels: labels,
115-
},
116-
},
117-
}
118-
}
119-
12081
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
12182

12283
// MongoDBList contains a list of MongoDB

pkg/controller/mongodb/mongodb_controller.go

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -213,8 +213,9 @@ func (r ReplicaSetReconciler) buildAutomationConfigConfigMap(mdb mdbv1.MongoDB)
213213
Build(), nil
214214
}
215215

216-
// buildContainers has some docs.
217-
func buildContainers(mdb mdbv1.MongoDB) ([]corev1.Container, error) {
216+
// buildContainers constructs the mongodb-agent container as well as the
217+
// mongod container.
218+
func buildContainers(mdb mdbv1.MongoDB) []corev1.Container {
218219
agentCommand := []string{
219220
"agent/mongodb-agent",
220221
"-cluster=" + clusterFilePath,
@@ -245,7 +246,7 @@ func buildContainers(mdb mdbv1.MongoDB) ([]corev1.Container, error) {
245246
Command: mongoDbCommand,
246247
Resources: resourcerequirements.Defaults(),
247248
}
248-
return []corev1.Container{agentContainer, mongodbContainer}, nil
249+
return []corev1.Container{agentContainer, mongodbContainer}
249250
}
250251

251252
func defaultReadinessProbe() corev1.Probe {
@@ -268,17 +269,12 @@ func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) {
268269
"app": mdb.ServiceName(),
269270
}
270271

271-
containers, err := buildContainers(mdb)
272-
if err != nil {
273-
return appsv1.StatefulSet{}, fmt.Errorf("error creating containers for %s/%s: %s", mdb.Namespace, mdb.Name, err)
274-
}
275-
276272
podSpecTemplate := corev1.PodTemplateSpec{
277273
ObjectMeta: metav1.ObjectMeta{
278274
Labels: labels,
279275
},
280276
Spec: corev1.PodSpec{
281-
Containers: containers,
277+
Containers: buildContainers(mdb),
282278
},
283279
}
284280

@@ -303,7 +299,6 @@ func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) {
303299
AddVolumeMount(mongodbName, dataVolume).
304300
AddVolumeMount(agentName, dataVolume).
305301
AddVolumeClaimTemplates(dataVolumeClaim)
306-
307302
// the automation config is only mounted, as read only, on the agent container
308303
automationConfigVolume := statefulset.CreateVolumeFromConfigMap("automation-config", "example-mongodb-config")
309304
automationConfigVolumeMount := statefulset.CreateVolumeMount("automation-config", "/var/lib/automation/config", statefulset.WithReadOnly(true))

test/e2e/e2eutil.go

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,14 @@ import (
66
"testing"
77
"time"
88

9-
mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1"
10-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"k8s.io/apimachinery/pkg/api/errors"
1110

1211
"github.com/mongodb/mongodb-kubernetes-operator/pkg/apis"
12+
mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1"
1313
f "github.com/operator-framework/operator-sdk/pkg/test"
1414
appsv1 "k8s.io/api/apps/v1"
1515
corev1 "k8s.io/api/core/v1"
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1617
"k8s.io/apimachinery/pkg/runtime"
1718
"k8s.io/apimachinery/pkg/types"
1819
"k8s.io/apimachinery/pkg/util/wait"
@@ -28,8 +29,15 @@ func RegisterTypesWithFramework(newTypes ...runtime.Object) error {
2829
return nil
2930
}
3031

31-
func CreateRuntimeObject(obj runtime.Object, ctx *f.TestCtx) error {
32-
return f.Global.Client.Create(context.TODO(), obj, &f.CleanupOptions{TestContext: ctx})
32+
func CreateOrUpdateMongoDB(mdb *mdbv1.MongoDB, ctx *f.TestCtx) error {
33+
err := f.Global.Client.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &mdbv1.MongoDB{})
34+
if err != nil {
35+
if errors.IsNotFound(err) {
36+
return f.Global.Client.Create(context.TODO(), mdb, &f.CleanupOptions{TestContext: ctx})
37+
}
38+
return err
39+
}
40+
return f.Global.Client.Update(context.TODO(), mdb)
3341
}
3442

3543
// waitForConfigMapToExist waits until a ConfigMap of the given name exists
@@ -48,31 +56,25 @@ func WaitForStatefulSetToExist(stsName string, retryInterval, timeout time.Durat
4856

4957
// waitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name
5058
// have reached the ready status
51-
func WaitForStatefulSetToBeReady(t *testing.T, stsName string, retryInterval, timeout time.Duration) error {
52-
return waitForStatefulSetCondition(t, stsName, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
53-
return *sts.Spec.Replicas == sts.Status.ReadyReplicas
54-
})
55-
}
56-
57-
func WaitForStatefulSetToNotBeReady(t *testing.T, stsName string, retryInterval, timeout time.Duration) error {
58-
return waitForStatefulSetCondition(t, stsName, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
59-
return *sts.Spec.Replicas != sts.Status.ReadyReplicas
59+
func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration) error {
60+
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
61+
return sts.Status.ReadyReplicas == int32(mdb.Spec.Members)
6062
})
6163
}
6264

63-
func waitForStatefulSetCondition(t *testing.T, stsName string, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error {
64-
_, err := WaitForStatefulSetToExist(stsName, retryInterval, timeout)
65+
func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error {
66+
_, err := WaitForStatefulSetToExist(mdb.Name, retryInterval, timeout)
6567
if err != nil {
6668
return fmt.Errorf("error waiting for stateful set to be created: %s", err)
6769
}
6870

6971
sts := appsv1.StatefulSet{}
7072
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
71-
err = f.Global.Client.Get(context.TODO(), types.NamespacedName{Name: stsName, Namespace: f.Global.Namespace}, &sts)
73+
err = f.Global.Client.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: f.Global.Namespace}, &sts)
7274
if err != nil {
7375
return false, err
7476
}
75-
t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d\n", stsName, *sts.Spec.Replicas, sts.Status.ReadyReplicas)
77+
t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d\n", mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas)
7678
ready := condition(sts)
7779
return ready, nil
7880
})

test/e2e/mongodbtests/mongodbtests.go

Lines changed: 70 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,17 @@ import (
2222

2323
// StatefulSetIsReady ensures that the underlying stateful set
2424
// reaches the running state
25-
func StatefulSetIsReady(mdb mdbv1.MongoDB) func(t *testing.T) {
25+
func StatefulSetIsReady(mdb *mdbv1.MongoDB) func(t *testing.T) {
2626
return func(t *testing.T) {
27-
err := e2eutil.WaitForStatefulSetToBeReady(t, mdb.Name, time.Second*15, time.Minute*5)
27+
err := e2eutil.WaitForStatefulSetToBeReady(t, mdb, time.Second*15, time.Minute*5)
2828
if err != nil {
2929
t.Fatal(err)
3030
}
3131
t.Logf("StatefulSet %s/%s is ready!", mdb.Namespace, mdb.Name)
3232
}
3333
}
3434

35-
func AutomationConfigConfigMapExists(mdb mdbv1.MongoDB) func(t *testing.T) {
35+
func AutomationConfigConfigMapExists(mdb *mdbv1.MongoDB) func(t *testing.T) {
3636
return func(t *testing.T) {
3737
cm, err := e2eutil.WaitForConfigMapToExist(mdb.ConfigMapName(), time.Second*5, time.Minute*1)
3838
assert.NoError(t, err)
@@ -44,15 +44,18 @@ func AutomationConfigConfigMapExists(mdb mdbv1.MongoDB) func(t *testing.T) {
4444
}
4545
}
4646

47-
func CreateResource(mdb mdbv1.MongoDB, ctx *f.TestCtx) func(*testing.T) {
47+
// CreateOrUpdateResource creates the MongoDB resource if it doesn't exist, or updates it otherwise
48+
func CreateOrUpdateResource(mdb *mdbv1.MongoDB, ctx *f.TestCtx) func(*testing.T) {
4849
return func(t *testing.T) {
49-
err := e2eutil.CreateRuntimeObject(&mdb, ctx)
50-
assert.NoError(t, err)
50+
if err := e2eutil.CreateOrUpdateMongoDB(mdb, ctx); err != nil {
51+
t.Fatal(err)
52+
}
5153
t.Logf("Created MongoDB resource %s/%s", mdb.Name, mdb.Namespace)
5254
}
5355
}
5456

55-
func DeletePod(mdb mdbv1.MongoDB, podNum int) func(*testing.T) {
57+
// DeletePod will delete a pod that belongs to this MongoDB resource's StatefulSet
58+
func DeletePod(mdb *mdbv1.MongoDB, podNum int) func(*testing.T) {
5659
return func(t *testing.T) {
5760
pod := corev1.Pod{
5861
ObjectMeta: metav1.ObjectMeta{
@@ -68,32 +71,72 @@ func DeletePod(mdb mdbv1.MongoDB, podNum int) func(*testing.T) {
6871
}
6972
}
7073

71-
// BasicConnectivity performs a check by initializing a mongo client
72-
// and inserting a document into the MongoDB resource
73-
func BasicConnectivity(mdb mdbv1.MongoDB) func(t *testing.T) {
74+
// BasicConnectivity returns a test function which performs
75+
// a basic MongoDB connectivity test
76+
func BasicConnectivity(mdb *mdbv1.MongoDB) func(t *testing.T) {
7477
return func(t *testing.T) {
75-
ctx, _ := context.WithTimeout(context.Background(), 10*time.Minute)
76-
mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(mdb.MongoURI()))
77-
if err != nil {
78-
t.Fatal(err)
78+
if err := Connect(mdb); err != nil {
79+
t.Fatal(fmt.Sprintf("Error connecting to MongoDB deployment: %+v", err))
7980
}
81+
t.Logf("successfully connected to MongoDB deployment")
82+
}
83+
}
8084

81-
t.Logf("Created mongo client!")
82-
83-
var res *mongo.InsertOneResult
84-
err = wait.Poll(time.Second*5, time.Minute*1, func() (done bool, err error) {
85-
collection := mongoClient.Database("testing").Collection("numbers")
86-
res, err = collection.InsertOne(ctx, bson.M{"name": "pi", "value": 3.14159})
87-
if err != nil {
88-
t.Logf("error inserting document: %+v", err)
89-
return false, err
90-
}
91-
return true, nil
92-
})
85+
// Connect performs a connectivity check by initializing a mongo client
86+
// and inserting a document into the MongoDB resource
87+
func Connect(mdb *mdbv1.MongoDB) error {
88+
ctx, _ := context.WithTimeout(context.Background(), 10*time.Minute)
89+
mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(mdb.MongoURI()))
90+
if err != nil {
91+
return err
92+
}
9393

94+
return wait.Poll(time.Second*5, time.Minute*2, func() (done bool, err error) {
95+
collection := mongoClient.Database("testing").Collection("numbers")
96+
_, err = collection.InsertOne(ctx, bson.M{"name": "pi", "value": 3.14159})
9497
if err != nil {
98+
return false, nil
99+
}
100+
return true, nil
101+
})
102+
}
103+
104+
// Scale update the MongoDB with a new number of members and updates the resource
105+
func Scale(mdb *mdbv1.MongoDB, newMembers int, ctx *f.TestCtx) func(*testing.T) {
106+
return func(t *testing.T) {
107+
mdb.Spec.Members = newMembers
108+
t.Logf("Scaling Mongodb %s, to %d members", mdb.Name, mdb.Spec.Members)
109+
if err := e2eutil.CreateOrUpdateMongoDB(mdb, ctx); err != nil {
95110
t.Fatal(err)
96111
}
97-
t.Logf("inserted ID: %+v", res.InsertedID)
112+
}
113+
}
114+
115+
// IsReachableDuring periodically tests connectivity to the provided MongoDB resource
116+
// during execution of the provided functions. This function can be used to ensure
117+
// The MongoDB is up throughout the test.
118+
func IsReachableDuring(mdb *mdbv1.MongoDB, interval time.Duration, testFunc func()) func(*testing.T) {
119+
return func(t *testing.T) {
120+
ctx, cancelFunc := context.WithCancel(context.Background())
121+
defer cancelFunc()
122+
123+
// start a go routine which will periodically check basic MongoDB connectivity
124+
// once all the test functions have been executed, the go routine will be cancelled
125+
go func() {
126+
for {
127+
select {
128+
case <-ctx.Done():
129+
t.Logf("context cancelled, no longer checking connectivity")
130+
return
131+
case <-time.After(interval):
132+
if err := Connect(mdb); err != nil {
133+
t.Fatal(fmt.Sprintf("error reaching MongoDB deployment: %+v", err))
134+
} else {
135+
t.Logf("Successfully connected to %s", mdb.Name)
136+
}
137+
}
138+
}
139+
}()
140+
testFunc()
98141
}
99142
}

test/e2e/replica_set/replica_set_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ func TestReplicaSet(t *testing.T) {
2323
}
2424

2525
mdb := e2eutil.NewTestMongoDB()
26-
t.Run("Create MongoDB Resource", mongodbtests.CreateResource(mdb, ctx))
27-
t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(mdb))
28-
t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(mdb))
29-
t.Run("Test Basic Connectivity", mongodbtests.BasicConnectivity(mdb))
26+
t.Run("Create MongoDB Resource", mongodbtests.CreateOrUpdateResource(&mdb, ctx))
27+
t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb))
28+
t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb))
29+
t.Run("Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb))
3030
}

0 commit comments

Comments
 (0)