Skip to content

Commit 2d2a258

Browse files
dkoshkindkoshkin
committed
fixup! fix: actually use the management cluster
1 parent bb42c20 commit 2d2a258

File tree

7 files changed

+379
-78
lines changed

7 files changed

+379
-78
lines changed

‎common/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ require (
1919
k8s.io/api v0.32.5
2020
k8s.io/apiextensions-apiserver v0.32.5
2121
k8s.io/apimachinery v0.32.5
22+
k8s.io/client-go v0.32.5
2223
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
2324
sigs.k8s.io/cluster-api v1.10.2
2425
sigs.k8s.io/cluster-api/test v1.10.2
@@ -86,7 +87,6 @@ require (
8687
gopkg.in/inf.v0 v0.9.1 // indirect
8788
gopkg.in/yaml.v3 v3.0.1 // indirect
8889
k8s.io/apiserver v0.32.5 // indirect
89-
k8s.io/client-go v0.32.5 // indirect
9090
k8s.io/cluster-bootstrap v0.32.3 // indirect
9191
k8s.io/component-base v0.32.5 // indirect
9292
k8s.io/klog/v2 v2.130.1 // indirect

‎common/pkg/capi/utils/utils.go‎

Lines changed: 85 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,37 +16,109 @@ import (
1616

1717
// ManagementCluster returns a Cluster object if c is pointing to a management cluster, otherwise returns nil.
1818
func ManagementCluster(ctx context.Context, c client.Reader) (*clusterv1.Cluster, error) {
19-
allNodes := &corev1.NodeList{}
20-
err := c.List(ctx, allNodes)
19+
clusterName, clusterNamespace, err := clusterAnnotationsFromNodes(ctx, c)
2120
if err != nil {
22-
return nil, fmt.Errorf("error listing Nodes: %w", err)
23-
}
24-
if len(allNodes.Items) == 0 {
25-
return nil, nil
21+
return nil, fmt.Errorf("error getting cluster annotations from Nodes: %w", err)
2622
}
27-
annotations := allNodes.Items[0].Annotations
28-
clusterName := annotations[clusterv1.ClusterNameAnnotation]
29-
clusterNamespace := annotations[clusterv1.ClusterNamespaceAnnotation]
23+
3024
if clusterName == "" && clusterNamespace == "" {
3125
return nil, nil
3226
}
3327

28+
cluster, err := managementClusterFromNodeAnnotations(ctx, c, clusterName, clusterNamespace)
29+
if err != nil {
30+
if k8serrors.IsNotFound(err) || meta.IsNoMatchError(err) {
31+
return nil, nil
32+
}
33+
return nil, err
34+
}
35+
36+
return cluster, nil
37+
}
38+
39+
// ManagementOrFutureManagementCluster returns a Cluster object to either the management cluster,
40+
// when c is a client to the management cluster.
41+
// Or a cluster that is assumed to become the management cluster in the future,
42+
// when is c is a client to a bootstrap cluster and there is only a single Cluster object.
43+
func ManagementOrFutureManagementCluster(ctx context.Context, c client.Reader) (*clusterv1.Cluster, error) {
44+
clusterName, clusterNamespace, err := clusterAnnotationsFromNodes(ctx, c)
45+
if err != nil {
46+
return nil, fmt.Errorf("error getting cluster annotations from Nodes: %w", err)
47+
}
48+
49+
var cluster *clusterv1.Cluster
50+
switch {
51+
case clusterName != "" && clusterNamespace != "":
52+
cluster, err = managementClusterFromNodeAnnotations(ctx, c, clusterName, clusterNamespace)
53+
case clusterName == "" && clusterNamespace == "":
54+
cluster, err = managementClusterFromBootstrapClient(ctx, c)
55+
case clusterName == "":
56+
err = fmt.Errorf("missing %q annotation", clusterv1.ClusterNameAnnotation)
57+
case clusterNamespace == "":
58+
err = fmt.Errorf("missing %q annotation", clusterv1.ClusterNamespaceAnnotation)
59+
}
60+
if err != nil {
61+
return nil, fmt.Errorf("error determining management cluster for the provided client: %w", err)
62+
}
63+
64+
return cluster, nil
65+
}
66+
67+
func clusterAnnotationsFromNodes(ctx context.Context, c client.Reader) (string, string, error) {
68+
allNodes := &corev1.NodeList{}
69+
err := c.List(ctx, allNodes)
70+
if err != nil {
71+
return "", "", fmt.Errorf("error listing Nodes: %w", err)
72+
}
73+
74+
// Get node annotations that should exist in the management cluster.
75+
annotations := make(map[string]string)
76+
if len(allNodes.Items) > 0 {
77+
annotations = allNodes.Items[0].Annotations
78+
}
79+
return annotations[clusterv1.ClusterNameAnnotation], annotations[clusterv1.ClusterNamespaceAnnotation], nil
80+
}
81+
82+
func managementClusterFromNodeAnnotations(
83+
ctx context.Context,
84+
c client.Reader,
85+
clusterName, clusterNamespace string,
86+
) (*clusterv1.Cluster, error) {
3487
cluster := &clusterv1.Cluster{}
3588
key := client.ObjectKey{
3689
Name: clusterName,
3790
Namespace: clusterNamespace,
3891
}
39-
err = c.Get(ctx, key, cluster)
92+
err := c.Get(ctx, key, cluster)
4093
if err != nil {
41-
if k8serrors.IsNotFound(err) || meta.IsNoMatchError(err) {
42-
return nil, nil
43-
}
4494
return nil, fmt.Errorf("error getting Cluster object based on Node annotations: %w", err)
4595
}
4696

4797
return cluster, nil
4898
}
4999

100+
// managementClusterFromBootstrapClient returns a Cluster object that is assumed to become the management cluster.
101+
// Returns an error if there is not exactly one Cluster object in the cluster.
102+
func managementClusterFromBootstrapClient(
103+
ctx context.Context,
104+
c client.Reader,
105+
) (*clusterv1.Cluster, error) {
106+
clusters := &clusterv1.ClusterList{}
107+
err := c.List(ctx, clusters)
108+
if err != nil {
109+
return nil, fmt.Errorf("error listing Clusters: %w", err)
110+
}
111+
112+
switch {
113+
case len(clusters.Items) == 0:
114+
return nil, fmt.Errorf("no Cluster objects found")
115+
case len(clusters.Items) > 1:
116+
return nil, fmt.Errorf("multiple Cluster objects found, expected exactly one")
117+
default:
118+
return &clusters.Items[0], nil
119+
}
120+
}
121+
50122
func GetProvider(cluster *clusterv1.Cluster) string {
51123
if cluster == nil {
52124
return ""

‎common/pkg/capi/utils/utils_test.go‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package utils
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
corev1 "k8s.io/api/core/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/runtime"
16+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
17+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
18+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
19+
"sigs.k8s.io/controller-runtime/pkg/client"
20+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
21+
)
22+
23+
func TestManagementOrFutureManagementCluster(t *testing.T) {
24+
t.Parallel()
25+
tests := []struct {
26+
name string
27+
initialClusters []clusterv1.Cluster
28+
nodes []corev1.Node
29+
wantCluster *clusterv1.Cluster
30+
wantErr error
31+
}{
32+
{
33+
name: "management cluster from Node annotations",
34+
initialClusters: []clusterv1.Cluster{
35+
{
36+
ObjectMeta: metav1.ObjectMeta{
37+
Name: "management-cluster",
38+
Namespace: "default",
39+
},
40+
},
41+
},
42+
nodes: []corev1.Node{
43+
{
44+
ObjectMeta: metav1.ObjectMeta{
45+
Name: "node1",
46+
Annotations: map[string]string{
47+
clusterv1.ClusterNameAnnotation: "management-cluster",
48+
clusterv1.ClusterNamespaceAnnotation: "default",
49+
},
50+
},
51+
},
52+
},
53+
wantCluster: &clusterv1.Cluster{
54+
ObjectMeta: metav1.ObjectMeta{
55+
Name: "management-cluster",
56+
Namespace: "default",
57+
},
58+
},
59+
},
60+
{
61+
name: "management cluster from Node annotations with multiple clusters",
62+
initialClusters: []clusterv1.Cluster{
63+
{
64+
ObjectMeta: metav1.ObjectMeta{
65+
Name: "management-cluster",
66+
Namespace: "default",
67+
},
68+
},
69+
{
70+
ObjectMeta: metav1.ObjectMeta{
71+
Name: "workload-cluster",
72+
Namespace: "default",
73+
},
74+
},
75+
},
76+
nodes: []corev1.Node{
77+
{
78+
ObjectMeta: metav1.ObjectMeta{
79+
Name: "node1",
80+
Annotations: map[string]string{
81+
clusterv1.ClusterNameAnnotation: "management-cluster",
82+
clusterv1.ClusterNamespaceAnnotation: "default",
83+
},
84+
},
85+
},
86+
},
87+
wantCluster: &clusterv1.Cluster{
88+
ObjectMeta: metav1.ObjectMeta{
89+
Name: "management-cluster",
90+
Namespace: "default",
91+
},
92+
},
93+
},
94+
{
95+
name: "management cluster from bootstrap client with single cluster",
96+
initialClusters: []clusterv1.Cluster{
97+
{
98+
ObjectMeta: metav1.ObjectMeta{
99+
Name: "management-cluster",
100+
Namespace: "default",
101+
},
102+
},
103+
},
104+
nodes: []corev1.Node{
105+
{
106+
ObjectMeta: metav1.ObjectMeta{
107+
Name: "node1",
108+
},
109+
},
110+
},
111+
wantCluster: &clusterv1.Cluster{
112+
ObjectMeta: metav1.ObjectMeta{
113+
Name: "management-cluster",
114+
Namespace: "default",
115+
},
116+
},
117+
},
118+
{
119+
name: "fail on missing cluster",
120+
// Cluster is in the wrong namespace.
121+
initialClusters: []clusterv1.Cluster{
122+
{
123+
ObjectMeta: metav1.ObjectMeta{
124+
Name: "management-cluster",
125+
Namespace: "different-namespace",
126+
},
127+
},
128+
},
129+
nodes: []corev1.Node{
130+
{
131+
ObjectMeta: metav1.ObjectMeta{
132+
Name: "node1",
133+
Annotations: map[string]string{
134+
clusterv1.ClusterNameAnnotation: "management-cluster",
135+
clusterv1.ClusterNamespaceAnnotation: "default",
136+
},
137+
},
138+
},
139+
},
140+
wantErr: fmt.Errorf(
141+
"error determining management cluster for the provided client: error getting Cluster object based on Node annotations: clusters.cluster.x-k8s.io \"management-cluster\" not found",
142+
),
143+
},
144+
{
145+
name: "fail on missing cluster name annotation",
146+
initialClusters: []clusterv1.Cluster{
147+
{
148+
ObjectMeta: metav1.ObjectMeta{
149+
Name: "management-cluster",
150+
Namespace: "default",
151+
},
152+
},
153+
},
154+
nodes: []corev1.Node{
155+
{
156+
ObjectMeta: metav1.ObjectMeta{
157+
Name: "node1",
158+
Annotations: map[string]string{
159+
clusterv1.ClusterNamespaceAnnotation: "default",
160+
},
161+
},
162+
},
163+
},
164+
wantErr: fmt.Errorf(
165+
"error determining management cluster for the provided client: missing \"cluster.x-k8s.io/cluster-name\" annotation",
166+
),
167+
},
168+
{
169+
name: "fail on missing cluster namespace annotation",
170+
initialClusters: []clusterv1.Cluster{
171+
{
172+
ObjectMeta: metav1.ObjectMeta{
173+
Name: "management-cluster",
174+
Namespace: "default",
175+
},
176+
},
177+
},
178+
nodes: []corev1.Node{
179+
{
180+
ObjectMeta: metav1.ObjectMeta{
181+
Name: "node1",
182+
Annotations: map[string]string{
183+
clusterv1.ClusterNameAnnotation: "management-cluster",
184+
},
185+
},
186+
},
187+
},
188+
wantErr: fmt.Errorf(
189+
"error determining management cluster for the provided client: missing \"cluster.x-k8s.io/cluster-namespace\" annotation",
190+
),
191+
},
192+
{
193+
name: "fail when multiple clusters exist in bootstrap client",
194+
initialClusters: []clusterv1.Cluster{
195+
{
196+
ObjectMeta: metav1.ObjectMeta{
197+
Name: "management-cluster",
198+
Namespace: "default",
199+
},
200+
},
201+
{
202+
ObjectMeta: metav1.ObjectMeta{
203+
Name: "another-management-cluster",
204+
Namespace: "default",
205+
},
206+
},
207+
},
208+
nodes: []corev1.Node{
209+
{
210+
ObjectMeta: metav1.ObjectMeta{
211+
Name: "node1",
212+
},
213+
},
214+
},
215+
wantErr: fmt.Errorf(
216+
"error determining management cluster for the provided client: multiple Cluster objects found, expected exactly one",
217+
),
218+
},
219+
}
220+
221+
for _, tt := range tests {
222+
t.Run(tt.name, func(t *testing.T) {
223+
t.Parallel()
224+
cl := buildFakeClientForTest(t, tt.initialClusters, tt.nodes)
225+
226+
cluster, err := ManagementOrFutureManagementCluster(context.Background(), cl)
227+
if tt.wantErr != nil {
228+
require.EqualError(t, err, tt.wantErr.Error())
229+
} else {
230+
require.NoError(t, err)
231+
}
232+
if tt.wantCluster != nil {
233+
assert.Equal(t, tt.wantCluster.GetName(), cluster.GetName())
234+
assert.Equal(t, tt.wantCluster.GetNamespace(), cluster.GetNamespace())
235+
}
236+
})
237+
}
238+
}
239+
240+
func buildFakeClientForTest(t *testing.T, clusters []clusterv1.Cluster, nodes []corev1.Node) client.Client {
241+
t.Helper()
242+
var objs []client.Object
243+
for i := range clusters {
244+
objs = append(objs, &clusters[i])
245+
}
246+
for i := range nodes {
247+
objs = append(objs, &nodes[i])
248+
}
249+
return buildFakeClient(t, objs...)
250+
}
251+
252+
func buildFakeClient(t *testing.T, objs ...client.Object) client.Client {
253+
t.Helper()
254+
clientScheme := runtime.NewScheme()
255+
utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
256+
utilruntime.Must(clusterv1.AddToScheme(clientScheme))
257+
return fake.NewClientBuilder().WithScheme(clientScheme).WithObjects(objs...).Build()
258+
}

0 commit comments

Comments
 (0)