Skip to content

Commit 093b1fa

Browse files
ntnnntnn
committed
Add tests
1 parent e071c84 commit 093b1fa

File tree

2 files changed

+373
-0
lines changed

2 files changed

+373
-0
lines changed

‎providers/multi/multi_suite_test.go

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package multi
18+
19+
import (
20+
"testing"
21+
22+
"k8s.io/client-go/rest"
23+
24+
"sigs.k8s.io/controller-runtime/pkg/envtest"
25+
logf "sigs.k8s.io/controller-runtime/pkg/log"
26+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
27+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
28+
29+
. "github.com/onsi/ginkgo/v2"
30+
. "github.com/onsi/gomega"
31+
)
32+
33+
func TestBuilder(t *testing.T) {
34+
RegisterFailHandler(Fail)
35+
RunSpecs(t, "Namespace Provider Suite")
36+
}
37+
38+
// The operator runs in a local cluster and embeds two other providers
39+
// for cloud providers. The cloud providers are simulated by using the
40+
// namespace provider with two other clusters.
41+
42+
var localEnv *envtest.Environment
43+
var localCfg *rest.Config
44+
45+
var cloud1 *envtest.Environment
46+
var cloud1cfg *rest.Config
47+
48+
var cloud2 *envtest.Environment
49+
var cloud2cfg *rest.Config
50+
51+
var _ = BeforeSuite(func() {
52+
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
53+
54+
var err error
55+
56+
localEnv = &envtest.Environment{}
57+
localCfg, err = localEnv.Start()
58+
Expect(err).NotTo(HaveOccurred())
59+
60+
cloud1 = &envtest.Environment{}
61+
cloud1cfg, err = cloud1.Start()
62+
Expect(err).NotTo(HaveOccurred())
63+
64+
cloud2 = &envtest.Environment{}
65+
cloud2cfg, err = cloud2.Start()
66+
Expect(err).NotTo(HaveOccurred())
67+
68+
// Prevent the metrics listener being created
69+
metricsserver.DefaultBindAddress = "0"
70+
})
71+
72+
var _ = AfterSuite(func() {
73+
if localEnv != nil {
74+
Expect(localEnv.Stop()).To(Succeed())
75+
}
76+
77+
if cloud1 != nil {
78+
Expect(cloud1.Stop()).To(Succeed())
79+
}
80+
81+
if cloud2 != nil {
82+
Expect(cloud2.Stop()).To(Succeed())
83+
}
84+
85+
// Put the DefaultBindAddress back
86+
metricsserver.DefaultBindAddress = ":8080"
87+
})

‎providers/multi/provider_test.go

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package multi
18+
19+
import (
20+
"context"
21+
"errors"
22+
"fmt"
23+
"strconv"
24+
25+
"golang.org/x/sync/errgroup"
26+
27+
corev1 "k8s.io/api/core/v1"
28+
apierrors "k8s.io/apimachinery/pkg/api/errors"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/apimachinery/pkg/util/runtime"
31+
"k8s.io/client-go/util/retry"
32+
33+
ctrl "sigs.k8s.io/controller-runtime"
34+
"sigs.k8s.io/controller-runtime/pkg/client"
35+
"sigs.k8s.io/controller-runtime/pkg/cluster"
36+
"sigs.k8s.io/controller-runtime/pkg/log"
37+
"sigs.k8s.io/controller-runtime/pkg/manager"
38+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
39+
40+
mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder"
41+
mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager"
42+
mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile"
43+
nsprovider "sigs.k8s.io/multicluster-runtime/providers/namespace"
44+
45+
. "github.com/onsi/ginkgo/v2"
46+
. "github.com/onsi/gomega"
47+
)
48+
49+
var _ = Describe("Provider Multi", Ordered, func() {
50+
ctx, cancel := context.WithCancel(context.Background())
51+
g, ctx := errgroup.WithContext(ctx)
52+
53+
var provider *Provider
54+
var mgr mcmanager.Manager
55+
var cloud1client, cloud2client client.Client
56+
var cloud1cluster, cloud2cluster cluster.Cluster
57+
var cloud1provider, cloud2provider *nsprovider.Provider
58+
59+
BeforeAll(func() {
60+
By("Setting up the first namespace provider", func() {
61+
var err error
62+
cloud1client, err = client.New(cloud1cfg, client.Options{})
63+
Expect(err).NotTo(HaveOccurred())
64+
65+
cloud1cluster, err = cluster.New(cloud1cfg)
66+
Expect(err).NotTo(HaveOccurred())
67+
g.Go(func() error {
68+
return ignoreCanceled(cloud1cluster.Start(ctx))
69+
})
70+
71+
cloud1provider = nsprovider.New(cloud1cluster)
72+
})
73+
74+
By("Setting up the second namespace provider", func() {
75+
var err error
76+
cloud2client, err = client.New(cloud2cfg, client.Options{})
77+
Expect(err).NotTo(HaveOccurred())
78+
79+
cloud2cluster, err = cluster.New(cloud2cfg)
80+
Expect(err).NotTo(HaveOccurred())
81+
g.Go(func() error {
82+
return ignoreCanceled(cloud2cluster.Start(ctx))
83+
})
84+
85+
cloud2provider = nsprovider.New(cloud2cluster)
86+
})
87+
88+
By("Setting up the provider and manager", func() {
89+
provider = New(Options{})
90+
91+
var err error
92+
mgr, err = mcmanager.New(localCfg, provider, manager.Options{})
93+
Expect(err).NotTo(HaveOccurred())
94+
95+
provider.SetManager(mgr)
96+
})
97+
98+
By("Adding the namespace providers to the multi provider", func() {
99+
// Without waiting for the cache sync adding the provider
100+
// will fail because the cache informer is not ready yet.
101+
cloud1cluster.GetCache().WaitForCacheSync(ctx)
102+
err := provider.AddProvider(ctx, "cloud1", cloud1provider, cloud1provider.Run)
103+
Expect(err).NotTo(HaveOccurred())
104+
105+
cloud2cluster.GetCache().WaitForCacheSync(ctx)
106+
err = provider.AddProvider(ctx, "cloud2", cloud2provider, cloud2provider.Run)
107+
Expect(err).NotTo(HaveOccurred())
108+
})
109+
110+
By("Setting up the controller feeding the animals", func() {
111+
err := mcbuilder.ControllerManagedBy(mgr).
112+
Named("fleet-ns-configmap-controller").
113+
For(&corev1.ConfigMap{}).
114+
Complete(mcreconcile.Func(
115+
func(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
116+
log := log.FromContext(ctx).WithValues("request", req.String())
117+
log.Info("Reconciling ConfigMap")
118+
119+
cl, err := mgr.GetCluster(ctx, req.ClusterName)
120+
if err != nil {
121+
return reconcile.Result{}, fmt.Errorf("failed to get cluster: %w", err)
122+
}
123+
124+
// Feed the animal.
125+
cm := &corev1.ConfigMap{}
126+
if err := cl.GetClient().Get(ctx, req.NamespacedName, cm); err != nil {
127+
if apierrors.IsNotFound(err) {
128+
return reconcile.Result{}, nil
129+
}
130+
return reconcile.Result{}, fmt.Errorf("failed to get configmap: %w", err)
131+
}
132+
if cm.GetLabels()["type"] != "animal" {
133+
return reconcile.Result{}, nil
134+
}
135+
136+
cm.Data = map[string]string{"stomach": "food"}
137+
if err := cl.GetClient().Update(ctx, cm); err != nil {
138+
return reconcile.Result{}, fmt.Errorf("failed to update configmap: %w", err)
139+
}
140+
141+
return ctrl.Result{}, nil
142+
},
143+
))
144+
Expect(err).NotTo(HaveOccurred())
145+
146+
By("Adding an index to the provider clusters", func() {
147+
err := mgr.GetFieldIndexer().IndexField(ctx, &corev1.ConfigMap{}, "type", func(obj client.Object) []string {
148+
return []string{obj.GetLabels()["type"]}
149+
})
150+
Expect(err).NotTo(HaveOccurred())
151+
})
152+
})
153+
154+
By("Starting the manager", func() {
155+
g.Go(func() error {
156+
return ignoreCanceled(mgr.Start(ctx))
157+
})
158+
})
159+
})
160+
161+
BeforeAll(func() {
162+
// cluster zoo exists in cloud1
163+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "zoo"}})))
164+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "zoo", Name: "elephant", Labels: map[string]string{"type": "animal"}}})))
165+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "zoo", Name: "lion", Labels: map[string]string{"type": "animal"}}})))
166+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "zoo", Name: "keeper", Labels: map[string]string{"type": "human"}}})))
167+
168+
// cluster jungle exists in cloud2
169+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "jungle"}})))
170+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "jungle", Name: "monkey", Labels: map[string]string{"type": "animal"}}})))
171+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "jungle", Name: "tree", Labels: map[string]string{"type": "thing"}}})))
172+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "jungle", Name: "tarzan", Labels: map[string]string{"type": "human"}}})))
173+
174+
// cluster island exists in both clouds
175+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "island"}})))
176+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "bird", Labels: map[string]string{"type": "animal"}}})))
177+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "stone", Labels: map[string]string{"type": "thing"}}})))
178+
runtime.Must(client.IgnoreAlreadyExists(cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "crusoe", Labels: map[string]string{"type": "human"}}})))
179+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "island"}})))
180+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "bird", Labels: map[string]string{"type": "animal"}}})))
181+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "stone", Labels: map[string]string{"type": "thing"}}})))
182+
runtime.Must(client.IgnoreAlreadyExists(cloud2client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "island", Name: "selkirk", Labels: map[string]string{"type": "human"}}})))
183+
})
184+
185+
It("runs the reconciler for existing objects", func(ctx context.Context) {
186+
Eventually(func() string {
187+
cl, err := mgr.GetCluster(ctx, "cloud1#zoo")
188+
Expect(err).NotTo(HaveOccurred())
189+
lion := &corev1.ConfigMap{}
190+
err = cl.GetClient().Get(ctx, client.ObjectKey{Namespace: "default", Name: "lion"}, lion)
191+
Expect(err).NotTo(HaveOccurred())
192+
return lion.Data["stomach"]
193+
}, "10s").Should(Equal("food"))
194+
})
195+
196+
It("runs the reconciler for new objects", func(ctx context.Context) {
197+
By("Creating a new configmap", func() {
198+
err := cloud1client.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "zoo", Name: "tiger", Labels: map[string]string{"type": "animal"}}})
199+
Expect(err).NotTo(HaveOccurred())
200+
})
201+
202+
Eventually(func() string {
203+
cl, err := mgr.GetCluster(ctx, "cloud1#zoo")
204+
Expect(err).NotTo(HaveOccurred())
205+
tiger := &corev1.ConfigMap{}
206+
err = cl.GetClient().Get(ctx, client.ObjectKey{Namespace: "default", Name: "tiger"}, tiger)
207+
Expect(err).NotTo(HaveOccurred())
208+
return tiger.Data["stomach"]
209+
}, "10s").Should(Equal("food"))
210+
})
211+
212+
It("runs the reconciler for updated objects", func(ctx context.Context) {
213+
updated := &corev1.ConfigMap{}
214+
By("Emptying the elephant's stomach", func() {
215+
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
216+
if err := cloud1client.Get(ctx, client.ObjectKey{Namespace: "zoo", Name: "elephant"}, updated); err != nil {
217+
return err
218+
}
219+
updated.Data = map[string]string{}
220+
return cloud1client.Update(ctx, updated)
221+
})
222+
Expect(err).NotTo(HaveOccurred())
223+
})
224+
rv, err := strconv.ParseInt(updated.ResourceVersion, 10, 64)
225+
Expect(err).NotTo(HaveOccurred())
226+
227+
Eventually(func() int64 {
228+
cl, err := mgr.GetCluster(ctx, "cloud1#zoo")
229+
Expect(err).NotTo(HaveOccurred())
230+
elephant := &corev1.ConfigMap{}
231+
err = cl.GetClient().Get(ctx, client.ObjectKey{Namespace: "default", Name: "elephant"}, elephant)
232+
Expect(err).NotTo(HaveOccurred())
233+
rv, err := strconv.ParseInt(elephant.ResourceVersion, 10, 64)
234+
Expect(err).NotTo(HaveOccurred())
235+
return rv
236+
}, "10s").Should(BeNumerically(">=", rv))
237+
238+
Eventually(func() string {
239+
cl, err := mgr.GetCluster(ctx, "cloud1#zoo")
240+
Expect(err).NotTo(HaveOccurred())
241+
elephant := &corev1.ConfigMap{}
242+
err = cl.GetClient().Get(ctx, client.ObjectKey{Namespace: "default", Name: "elephant"}, elephant)
243+
Expect(err).NotTo(HaveOccurred())
244+
return elephant.Data["stomach"]
245+
}, "10s").Should(Equal("food"))
246+
})
247+
248+
It("queries island on cloud1 via a multi-cluster index", func() {
249+
island, err := mgr.GetCluster(ctx, "cloud1#island")
250+
Expect(err).NotTo(HaveOccurred())
251+
cms := &corev1.ConfigMapList{}
252+
err = island.GetCache().List(ctx, cms, client.MatchingFields{"type": "human"})
253+
Expect(err).NotTo(HaveOccurred())
254+
Expect(cms.Items).To(HaveLen(1))
255+
Expect(cms.Items[0].Name).To(Equal("crusoe"))
256+
Expect(cms.Items[0].Namespace).To(Equal("default"))
257+
})
258+
259+
It("queries island on cloud2 via a multi-cluster index", func() {
260+
island, err := mgr.GetCluster(ctx, "cloud2#island")
261+
Expect(err).NotTo(HaveOccurred())
262+
cms := &corev1.ConfigMapList{}
263+
err = island.GetCache().List(ctx, cms, client.MatchingFields{"type": "human"})
264+
Expect(err).NotTo(HaveOccurred())
265+
Expect(cms.Items).To(HaveLen(1))
266+
Expect(cms.Items[0].Name).To(Equal("selkirk"))
267+
Expect(cms.Items[0].Namespace).To(Equal("default"))
268+
})
269+
270+
AfterAll(func() {
271+
By("Stopping the provider, cluster, manager, and controller", func() {
272+
cancel()
273+
})
274+
By("Waiting for the error group to finish", func() {
275+
err := g.Wait()
276+
Expect(err).NotTo(HaveOccurred())
277+
})
278+
})
279+
})
280+
281+
func ignoreCanceled(err error) error {
282+
if errors.Is(err, context.Canceled) {
283+
return nil
284+
}
285+
return err
286+
}

0 commit comments

Comments
 (0)