Skip to content

Commit 7488ec9

Browse files
tjamettjamet
committed
Add an optional reconciler to watch for ConfigSecret changes
Currently, if a ConfigSecret is updated, it is not reflected automatically to all providers using it. Implement this behaviour, under an opt-in flag.
1 parent 52aef23 commit 7488ec9

File tree

5 files changed

+281
-2
lines changed

5 files changed

+281
-2
lines changed

‎cmd/main.go

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import (
4646
operatorv1alpha1 "sigs.k8s.io/cluster-api-operator/api/v1alpha1"
4747
operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
4848
providercontroller "sigs.k8s.io/cluster-api-operator/internal/controller"
49+
"sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider"
4950
healtchcheckcontroller "sigs.k8s.io/cluster-api-operator/internal/controller/healthcheck"
5051
)
5152

@@ -67,6 +68,7 @@ var (
6768
webhookPort int
6869
webhookCertDir string
6970
healthAddr string
71+
watchConfigSecretChanges bool
7072
diagnosticsOptions = flags.DiagnosticsOptions{}
7173
)
7274

@@ -98,6 +100,9 @@ func InitFlags(fs *pflag.FlagSet) {
98100
fs.StringVar(&watchFilterValue, "watch-filter", "",
99101
fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel))
100102

103+
fs.BoolVar(&watchConfigSecretChanges, "watch-configsecret", false,
104+
"Watch for changes to the ConfigSecret resource and reconcile all providers using it.")
105+
101106
fs.StringVar(&watchNamespace, "namespace", "",
102107
"Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.")
103108

@@ -185,7 +190,7 @@ func main() {
185190
ctx := ctrl.SetupSignalHandler()
186191

187192
setupChecks(mgr)
188-
setupReconcilers(mgr)
193+
setupReconcilers(mgr, watchConfigSecretChanges)
189194
setupWebhooks(mgr)
190195

191196
// +kubebuilder:scaffold:builder
@@ -209,7 +214,7 @@ func setupChecks(mgr ctrl.Manager) {
209214
}
210215
}
211216

212-
func setupReconcilers(mgr ctrl.Manager) {
217+
func setupReconcilers(mgr ctrl.Manager, watchConfigSecretChanges bool) {
213218
if err := (&providercontroller.GenericProviderReconciler{
214219
Provider: &operatorv1.CoreProvider{},
215220
ProviderList: &operatorv1.CoreProviderList{},
@@ -286,6 +291,24 @@ func setupReconcilers(mgr ctrl.Manager) {
286291
setupLog.Error(err, "unable to create controller", "controller", "Healthcheck")
287292
os.Exit(1)
288293
}
294+
295+
if watchConfigSecretChanges {
296+
if err := (&providercontroller.SecretReconciler{
297+
ProviderLists: []genericprovider.GenericProviderList{
298+
&operatorv1.CoreProviderList{},
299+
&operatorv1.InfrastructureProviderList{},
300+
&operatorv1.BootstrapProviderList{},
301+
&operatorv1.ControlPlaneProviderList{},
302+
&operatorv1.AddonProviderList{},
303+
&operatorv1.IPAMProviderList{},
304+
&operatorv1.RuntimeExtensionProviderList{},
305+
},
306+
Client: mgr.GetClient(),
307+
}).SetupWithManager(mgr, concurrency(concurrencyNumber)); err != nil {
308+
setupLog.Error(err, "unable to create controller", "controller", "Secret")
309+
os.Exit(1)
310+
}
311+
}
289312
}
290313

291314
func setupWebhooks(mgr ctrl.Manager) {

‎hack/charts/cluster-api-operator/templates/deployment.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ spec:
7474
{{- if .Values.insecureDiagnostics }}
7575
- --insecure-diagnostics={{ .Values.insecureDiagnostics }}
7676
{{- end }}
77+
{{- if .Values.watchConfigSecret }}
78+
- --watch-configsecret
79+
{{- end }}
7780
{{- with .Values.leaderElection }}
7881
- --leader-elect={{ .enabled }}
7982
{{- if .leaseDuration }}

‎hack/charts/cluster-api-operator/values.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ healthAddr: ":8081"
2727
metricsBindAddr: "127.0.0.1:8080"
2828
diagnosticsAddress: "8443"
2929
insecureDiagnostics: false
30+
watchConfigSecret: false
3031
imagePullSecrets: {}
3132
resources:
3233
manager:

‎internal/controller/secret_controller.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package controller
2+
3+
/*
4+
Copyright 2021 The Kubernetes Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
import (
20+
"context"
21+
22+
v1 "k8s.io/api/core/v1"
23+
apierrors "k8s.io/apimachinery/pkg/api/errors"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25+
"sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider"
26+
ctrl "sigs.k8s.io/controller-runtime"
27+
"sigs.k8s.io/controller-runtime/pkg/client"
28+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
29+
"sigs.k8s.io/controller-runtime/pkg/controller"
30+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
31+
)
32+
33+
type SecretReconciler struct {
34+
ProviderLists []genericprovider.GenericProviderList
35+
Client client.Client
36+
}
37+
38+
const (
39+
observedSpecHashAnnotation = "operator.cluster.x-k8s.io/observed-spec-hash"
40+
)
41+
42+
func (r *SecretReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
43+
return ctrl.NewControllerManagedBy(mgr).
44+
For(&v1.Secret{}).
45+
WithOptions(options).
46+
Complete(r)
47+
}
48+
49+
func (r *SecretReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, reterr error) {
50+
log := ctrl.LoggerFrom(ctx)
51+
52+
log.Info("Reconciling provider")
53+
54+
secret := &v1.Secret{
55+
ObjectMeta: metav1.ObjectMeta{
56+
Namespace: req.Namespace,
57+
Name: req.Name,
58+
},
59+
}
60+
61+
err := r.Client.Get(ctx, req.NamespacedName, secret)
62+
if ctrlclient.IgnoreNotFound(err) != nil {
63+
return reconcile.Result{}, err
64+
}
65+
66+
hash := ""
67+
if !apierrors.IsNotFound(err) {
68+
hash, err = calculateHash(secret.Data)
69+
if err != nil {
70+
return reconcile.Result{}, err
71+
}
72+
}
73+
74+
for _, group := range r.ProviderLists {
75+
g, ok := group.(client.ObjectList)
76+
if !ok {
77+
continue
78+
}
79+
80+
if err := r.Client.List(ctx, g); err != nil {
81+
return reconcile.Result{}, err
82+
}
83+
84+
for _, p := range group.GetItems() {
85+
if p.GetSpec().ConfigSecret != nil {
86+
configNamespace := p.GetSpec().ConfigSecret.Namespace
87+
if configNamespace == "" {
88+
configNamespace = p.GetNamespace()
89+
}
90+
if configNamespace == req.Namespace && p.GetSpec().ConfigSecret.Name == req.Name {
91+
patched, ok := p.DeepCopyObject().(client.Object)
92+
if !ok {
93+
// todo: just log
94+
} else {
95+
annotations := patched.GetAnnotations()
96+
if annotations == nil {
97+
annotations = map[string]string{}
98+
}
99+
if annotations[observedSpecHashAnnotation] != hash {
100+
annotations[observedSpecHashAnnotation] = hash
101+
patched.SetAnnotations(annotations)
102+
err := r.Client.Patch(ctx, patched, client.MergeFrom(p))
103+
if err != nil {
104+
return reconcile.Result{}, err
105+
}
106+
}
107+
}
108+
}
109+
}
110+
}
111+
}
112+
return reconcile.Result{}, nil
113+
}

‎internal/controller/secret_controller_test.go

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package controller
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
. "github.com/onsi/gomega"
8+
v1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/apimachinery/pkg/runtime"
11+
"k8s.io/apimachinery/pkg/types"
12+
operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
13+
"sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
16+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
17+
)
18+
19+
func TestSecretReconciler(t *testing.T) {
20+
ctx := ctx
21+
22+
g := NewWithT(t)
23+
scheme := runtime.NewScheme()
24+
g.Expect(v1.AddToScheme(scheme)).To(Succeed())
25+
g.Expect(operatorv1.AddToScheme(scheme)).To(Succeed())
26+
27+
providersUsingTheSecret := []client.Object{
28+
&operatorv1.AddonProvider{
29+
ObjectMeta: metav1.ObjectMeta{
30+
Namespace: "default",
31+
Name: "some-addon",
32+
},
33+
Spec: operatorv1.AddonProviderSpec{
34+
ProviderSpec: operatorv1.ProviderSpec{
35+
ConfigSecret: &operatorv1.SecretReference{
36+
Name: "secret-in-default-namespace",
37+
},
38+
},
39+
},
40+
},
41+
&operatorv1.ControlPlaneProvider{
42+
ObjectMeta: metav1.ObjectMeta{
43+
Namespace: "some-namespace",
44+
Name: "some-control-plane",
45+
},
46+
Spec: operatorv1.ControlPlaneProviderSpec{
47+
ProviderSpec: operatorv1.ProviderSpec{
48+
ConfigSecret: &operatorv1.SecretReference{
49+
Name: "secret-in-default-namespace",
50+
Namespace: "default",
51+
},
52+
},
53+
},
54+
},
55+
}
56+
providersNotUsingTheSecret := []client.Object{
57+
&operatorv1.InfrastructureProvider{
58+
ObjectMeta: metav1.ObjectMeta{
59+
Namespace: "some-namespace",
60+
Name: "some-infra",
61+
},
62+
},
63+
&operatorv1.IPAMProvider{
64+
ObjectMeta: metav1.ObjectMeta{
65+
Namespace: "default",
66+
Name: "some-addon",
67+
},
68+
Spec: operatorv1.IPAMProviderSpec{
69+
ProviderSpec: operatorv1.ProviderSpec{
70+
ConfigSecret: &operatorv1.SecretReference{
71+
Name: "other-secret-name",
72+
},
73+
},
74+
},
75+
},
76+
}
77+
78+
k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(providersUsingTheSecret...).WithObjects(providersNotUsingTheSecret...).Build()
79+
80+
r := &SecretReconciler{
81+
Client: k8sClient,
82+
ProviderLists: []genericprovider.GenericProviderList{
83+
&operatorv1.AddonProviderList{},
84+
&operatorv1.ControlPlaneProviderList{},
85+
&operatorv1.InfrastructureProviderList{},
86+
&operatorv1.IPAMProviderList{},
87+
&operatorv1.BootstrapProviderList{},
88+
},
89+
}
90+
91+
t.Run("When the secret does not exist", func(t *testing.T) {
92+
_, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "secret-in-default-namespace", Namespace: "default"}})
93+
g.Expect(err).ToNot(HaveOccurred())
94+
95+
assertProvidersAreAnnotatedWithSecretVersions(g, ctx, k8sClient, providersUsingTheSecret, providersNotUsingTheSecret, "")
96+
97+
t.Run("Any subsequent reconciliation is successful", func(t *testing.T) {
98+
_, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "secret-in-default-namespace", Namespace: "default"}})
99+
g.Expect(err).ToNot(HaveOccurred())
100+
101+
assertProvidersAreAnnotatedWithSecretVersions(g, ctx, k8sClient, providersUsingTheSecret, providersNotUsingTheSecret, "")
102+
})
103+
})
104+
105+
t.Run("When the secret exists", func(t *testing.T) {
106+
secret := &v1.Secret{
107+
ObjectMeta: metav1.ObjectMeta{
108+
Namespace: "default",
109+
Name: "secret-in-default-namespace",
110+
},
111+
Data: map[string][]byte{
112+
"key": []byte("value"),
113+
},
114+
}
115+
g.Expect(r.Client.Create(ctx, secret)).To(Succeed())
116+
117+
_, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "secret-in-default-namespace", Namespace: "default"}})
118+
g.Expect(err).ToNot(HaveOccurred())
119+
assertProvidersAreAnnotatedWithSecretVersions(g, ctx, k8sClient, providersUsingTheSecret, providersNotUsingTheSecret, "fed7a27106a07691449b9cf7f57536328004b134d358d8fcafaf6a2a06f99d50")
120+
121+
t.Run("Any subsequent reconciliation is successful", func(t *testing.T) {
122+
_, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "secret-in-default-namespace", Namespace: "default"}})
123+
g.Expect(err).ToNot(HaveOccurred())
124+
})
125+
assertProvidersAreAnnotatedWithSecretVersions(g, ctx, k8sClient, providersUsingTheSecret, providersNotUsingTheSecret, "fed7a27106a07691449b9cf7f57536328004b134d358d8fcafaf6a2a06f99d50")
126+
})
127+
}
128+
129+
func assertProvidersAreAnnotatedWithSecretVersions(g *WithT, ctx context.Context, k8sClient client.Client, providersUsingTheSecret, providersNotUsingTheSecret []client.Object, secretHash string) {
130+
131+
for _, provider := range providersUsingTheSecret {
132+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(provider), provider)).To(Succeed())
133+
g.Expect(provider.GetAnnotations()[observedSpecHashAnnotation]).To(Equal(secretHash))
134+
}
135+
for _, provider := range providersNotUsingTheSecret {
136+
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(provider), provider)).To(Succeed())
137+
g.Expect(provider.GetAnnotations()).NotTo(HaveKey(observedSpecHashAnnotation))
138+
}
139+
}

0 commit comments

Comments
 (0)