Skip to content

Commit 9a2d8cd

Browse files
chrischdichrischdi
authored
🐛 crs: use separate cache for partial metadata watches on secrets to include all secrets (kubernetes-sigs#10633)
* crs: use separate cache for partial metadata watches on secrets to include all secrets * review fixes * review fixes * exp/test: start cache * review fixes * review fixes * review fixes * review fixes * review fixes
1 parent cb490f6 commit 9a2d8cd

File tree

7 files changed

+189
-102
lines changed

7 files changed

+189
-102
lines changed

‎exp/addons/controllers/alias.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121

2222
ctrl "sigs.k8s.io/controller-runtime"
23+
"sigs.k8s.io/controller-runtime/pkg/cache"
2324
"sigs.k8s.io/controller-runtime/pkg/client"
2425
"sigs.k8s.io/controller-runtime/pkg/controller"
2526

@@ -36,12 +37,12 @@ type ClusterResourceSetReconciler struct {
3637
WatchFilterValue string
3738
}
3839

39-
func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
40+
func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options, partialSecretCache cache.Cache) error {
4041
return (&clusterresourcesets.ClusterResourceSetReconciler{
4142
Client: r.Client,
4243
Tracker: r.Tracker,
4344
WatchFilterValue: r.WatchFilterValue,
44-
}).SetupWithManager(ctx, mgr, options)
45+
}).SetupWithManager(ctx, mgr, options, partialSecretCache)
4546
}
4647

4748
// ClusterResourceSetBindingReconciler reconciles a ClusterResourceSetBinding object.

‎exp/addons/internal/controllers/clusterresourceset_controller.go

Lines changed: 55 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,13 @@ import (
3232
"k8s.io/klog/v2"
3333
ctrl "sigs.k8s.io/controller-runtime"
3434
"sigs.k8s.io/controller-runtime/pkg/builder"
35+
"sigs.k8s.io/controller-runtime/pkg/cache"
3536
"sigs.k8s.io/controller-runtime/pkg/client"
3637
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
3738
"sigs.k8s.io/controller-runtime/pkg/controller"
3839
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
3940
"sigs.k8s.io/controller-runtime/pkg/handler"
41+
"sigs.k8s.io/controller-runtime/pkg/source"
4042

4143
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
4244
"sigs.k8s.io/cluster-api/controllers/remote"
@@ -65,7 +67,7 @@ type ClusterResourceSetReconciler struct {
6567
WatchFilterValue string
6668
}
6769

68-
func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
70+
func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options, partialSecretCache cache.Cache) error {
6971
err := ctrl.NewControllerManagedBy(mgr).
7072
For(&addonsv1.ClusterResourceSet{}).
7173
Watches(
@@ -74,18 +76,26 @@ func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr
7476
).
7577
WatchesMetadata(
7678
&corev1.ConfigMap{},
77-
handler.EnqueueRequestsFromMapFunc(r.resourceToClusterResourceSet),
78-
builder.WithPredicates(
79-
resourcepredicates.ResourceCreateOrUpdate(ctrl.LoggerFrom(ctx)),
79+
handler.EnqueueRequestsFromMapFunc(
80+
resourceToClusterResourceSetFunc[client.Object](r.Client),
8081
),
81-
).
82-
WatchesMetadata(
83-
&corev1.Secret{},
84-
handler.EnqueueRequestsFromMapFunc(r.resourceToClusterResourceSet),
8582
builder.WithPredicates(
86-
resourcepredicates.ResourceCreateOrUpdate(ctrl.LoggerFrom(ctx)),
83+
resourcepredicates.TypedResourceCreateOrUpdate[client.Object](ctrl.LoggerFrom(ctx)),
8784
),
8885
).
86+
WatchesRawSource(source.Kind(
87+
partialSecretCache,
88+
&metav1.PartialObjectMetadata{
89+
TypeMeta: metav1.TypeMeta{
90+
Kind: "Secret",
91+
APIVersion: "v1",
92+
},
93+
},
94+
handler.TypedEnqueueRequestsFromMapFunc(
95+
resourceToClusterResourceSetFunc[*metav1.PartialObjectMetadata](r.Client),
96+
),
97+
resourcepredicates.TypedResourceCreateOrUpdate[*metav1.PartialObjectMetadata](ctrl.LoggerFrom(ctx)),
98+
)).
8999
WithOptions(options).
90100
WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)).
91101
Complete(r)
@@ -476,46 +486,48 @@ func (r *ClusterResourceSetReconciler) clusterToClusterResourceSet(ctx context.C
476486
return result
477487
}
478488

479-
// resourceToClusterResourceSet is mapper function that maps resources to ClusterResourceSet.
480-
func (r *ClusterResourceSetReconciler) resourceToClusterResourceSet(ctx context.Context, o client.Object) []ctrl.Request {
481-
result := []ctrl.Request{}
489+
// resourceToClusterResourceSetFunc returns a typed mapper function that maps resources to ClusterResourceSet.
490+
func resourceToClusterResourceSetFunc[T client.Object](ctrlClient client.Client) handler.TypedMapFunc[T] {
491+
return func(ctx context.Context, o T) []ctrl.Request {
492+
result := []ctrl.Request{}
482493

483-
// Add all ClusterResourceSet owners.
484-
for _, owner := range o.GetOwnerReferences() {
485-
if owner.Kind == "ClusterResourceSet" {
486-
name := client.ObjectKey{Namespace: o.GetNamespace(), Name: owner.Name}
487-
result = append(result, ctrl.Request{NamespacedName: name})
494+
// Add all ClusterResourceSet owners.
495+
for _, owner := range o.GetOwnerReferences() {
496+
if owner.Kind == "ClusterResourceSet" {
497+
name := client.ObjectKey{Namespace: o.GetNamespace(), Name: owner.Name}
498+
result = append(result, ctrl.Request{NamespacedName: name})
499+
}
488500
}
489-
}
490501

491-
// If there is any ClusterResourceSet owner, that means the resource is reconciled before,
492-
// and existing owners are the only matching ClusterResourceSets to this resource, so no need to return all ClusterResourceSets.
493-
if len(result) > 0 {
494-
return result
495-
}
502+
// If there is any ClusterResourceSet owner, that means the resource is reconciled before,
503+
// and existing owners are the only matching ClusterResourceSets to this resource, so no need to return all ClusterResourceSets.
504+
if len(result) > 0 {
505+
return result
506+
}
496507

497-
// Only core group is accepted as resources group
498-
if o.GetObjectKind().GroupVersionKind().Group != "" {
499-
return result
500-
}
508+
// Only core group is accepted as resources group
509+
if o.GetObjectKind().GroupVersionKind().Group != "" {
510+
return result
511+
}
501512

502-
crsList := &addonsv1.ClusterResourceSetList{}
503-
if err := r.Client.List(ctx, crsList, client.InNamespace(o.GetNamespace())); err != nil {
504-
return nil
505-
}
506-
objKind, err := apiutil.GVKForObject(o, r.Client.Scheme())
507-
if err != nil {
508-
return nil
509-
}
510-
for _, crs := range crsList.Items {
511-
for _, resource := range crs.Spec.Resources {
512-
if resource.Kind == objKind.Kind && resource.Name == o.GetName() {
513-
name := client.ObjectKey{Namespace: o.GetNamespace(), Name: crs.Name}
514-
result = append(result, ctrl.Request{NamespacedName: name})
515-
break
513+
crsList := &addonsv1.ClusterResourceSetList{}
514+
if err := ctrlClient.List(ctx, crsList, client.InNamespace(o.GetNamespace())); err != nil {
515+
return nil
516+
}
517+
objKind, err := apiutil.GVKForObject(o, ctrlClient.Scheme())
518+
if err != nil {
519+
return nil
520+
}
521+
for _, crs := range crsList.Items {
522+
for _, resource := range crs.Spec.Resources {
523+
if resource.Kind == objKind.Kind && resource.Name == o.GetName() {
524+
name := client.ObjectKey{Namespace: o.GetNamespace(), Name: crs.Name}
525+
result = append(result, ctrl.Request{NamespacedName: name})
526+
break
527+
}
516528
}
517529
}
518-
}
519530

520-
return result
531+
return result
532+
}
521533
}

‎exp/addons/internal/controllers/predicates/resource_predicates.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,17 @@ package predicates
1919

2020
import (
2121
"github.com/go-logr/logr"
22+
"sigs.k8s.io/controller-runtime/pkg/client"
2223
"sigs.k8s.io/controller-runtime/pkg/event"
2324
"sigs.k8s.io/controller-runtime/pkg/predicate"
2425
)
2526

26-
// ResourceCreateOrUpdate returns a predicate that returns true for create and update events.
27-
func ResourceCreateOrUpdate(_ logr.Logger) predicate.Funcs {
28-
return predicate.Funcs{
29-
CreateFunc: func(event.CreateEvent) bool { return true },
30-
UpdateFunc: func(event.UpdateEvent) bool { return true },
31-
DeleteFunc: func(event.DeleteEvent) bool { return false },
32-
GenericFunc: func(event.GenericEvent) bool { return false },
27+
// TypedResourceCreateOrUpdate returns a predicate that returns true for create and update events.
28+
func TypedResourceCreateOrUpdate[T client.Object](_ logr.Logger) predicate.TypedFuncs[T] {
29+
return predicate.TypedFuncs[T]{
30+
CreateFunc: func(event.TypedCreateEvent[T]) bool { return true },
31+
UpdateFunc: func(event.TypedUpdateEvent[T]) bool { return true },
32+
DeleteFunc: func(event.TypedDeleteEvent[T]) bool { return false },
33+
GenericFunc: func(event.TypedGenericEvent[T]) bool { return false },
3334
}
3435
}

‎exp/addons/internal/controllers/suite_test.go

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,13 @@ import (
2121
"fmt"
2222
"os"
2323
"testing"
24+
"time"
2425

2526
corev1 "k8s.io/api/core/v1"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/utils/ptr"
2629
ctrl "sigs.k8s.io/controller-runtime"
30+
"sigs.k8s.io/controller-runtime/pkg/cache"
2731
"sigs.k8s.io/controller-runtime/pkg/client"
2832
"sigs.k8s.io/controller-runtime/pkg/controller"
2933

@@ -46,6 +50,32 @@ func TestMain(m *testing.M) {
4650
}
4751

4852
setupReconcilers := func(ctx context.Context, mgr ctrl.Manager) {
53+
// Create partial cache analog to main.go.
54+
partialSecretCache, err := cache.New(mgr.GetConfig(), cache.Options{
55+
Scheme: mgr.GetScheme(),
56+
Mapper: mgr.GetRESTMapper(),
57+
HTTPClient: mgr.GetHTTPClient(),
58+
SyncPeriod: ptr.To(time.Minute * 10),
59+
DefaultTransform: func(in interface{}) (interface{}, error) {
60+
// Use DefaultTransform to drop objects we don't expect to get into this cache.
61+
obj, ok := in.(*metav1.PartialObjectMetadata)
62+
if !ok {
63+
panic(fmt.Sprintf("cache expected to only get PartialObjectMetadata, got %T", in))
64+
}
65+
if obj.GetObjectKind().GroupVersionKind() != corev1.SchemeGroupVersion.WithKind("Secret") {
66+
panic(fmt.Sprintf("cache expected to only get Secrets, got %s", obj.GetObjectKind()))
67+
}
68+
// Additionally strip managed fields.
69+
return cache.TransformStripManagedFields()(obj)
70+
},
71+
})
72+
if err != nil {
73+
panic(fmt.Sprintf("Failed to create cache for metadata only Secret watches: %v", err))
74+
}
75+
if err := mgr.Add(partialSecretCache); err != nil {
76+
panic(fmt.Sprintf("Failed to start cache for metadata only Secret watches: %v", err))
77+
}
78+
4979
tracker, err := remote.NewClusterCacheTracker(mgr, remote.ClusterCacheTrackerOptions{})
5080
if err != nil {
5181
panic(fmt.Sprintf("Failed to create new cluster cache tracker: %v", err))
@@ -55,7 +85,7 @@ func TestMain(m *testing.M) {
5585
Client: mgr.GetClient(),
5686
Tracker: tracker,
5787
}
58-
if err = reconciler.SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: 1}); err != nil {
88+
if err = reconciler.SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: 1}, partialSecretCache); err != nil {
5989
panic(fmt.Sprintf("Failed to set up cluster resource set reconciler: %v", err))
6090
}
6191
bindingReconciler := ClusterResourceSetBindingReconciler{

‎exp/runtime/controllers/alias.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121

2222
ctrl "sigs.k8s.io/controller-runtime"
23+
"sigs.k8s.io/controller-runtime/pkg/cache"
2324
"sigs.k8s.io/controller-runtime/pkg/client"
2425
"sigs.k8s.io/controller-runtime/pkg/controller"
2526

@@ -37,11 +38,11 @@ type ExtensionConfigReconciler struct {
3738
WatchFilterValue string
3839
}
3940

40-
func (r *ExtensionConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
41+
func (r *ExtensionConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options, partialSecretCache cache.Cache) error {
4142
return (&runtimecontrollers.Reconciler{
4243
Client: r.Client,
4344
APIReader: r.APIReader,
4445
RuntimeClient: r.RuntimeClient,
4546
WatchFilterValue: r.WatchFilterValue,
46-
}).SetupWithManager(ctx, mgr, options)
47+
}).SetupWithManager(ctx, mgr, options, partialSecretCache)
4748
}

‎exp/runtime/internal/controllers/extensionconfig_controller.go

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,16 @@ import (
2424
"github.com/pkg/errors"
2525
corev1 "k8s.io/api/core/v1"
2626
apierrors "k8s.io/apimachinery/pkg/api/errors"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2728
"k8s.io/apimachinery/pkg/types"
2829
kerrors "k8s.io/apimachinery/pkg/util/errors"
2930
ctrl "sigs.k8s.io/controller-runtime"
31+
"sigs.k8s.io/controller-runtime/pkg/cache"
3032
"sigs.k8s.io/controller-runtime/pkg/client"
3133
"sigs.k8s.io/controller-runtime/pkg/controller"
3234
"sigs.k8s.io/controller-runtime/pkg/handler"
3335
"sigs.k8s.io/controller-runtime/pkg/reconcile"
36+
"sigs.k8s.io/controller-runtime/pkg/source"
3437

3538
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3639
runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1"
@@ -59,13 +62,21 @@ type Reconciler struct {
5962
WatchFilterValue string
6063
}
6164

62-
func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
65+
func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options, partialSecretCache cache.Cache) error {
6366
err := ctrl.NewControllerManagedBy(mgr).
6467
For(&runtimev1.ExtensionConfig{}).
65-
WatchesMetadata(
66-
&corev1.Secret{},
67-
handler.EnqueueRequestsFromMapFunc(r.secretToExtensionConfig),
68-
).
68+
WatchesRawSource(source.Kind(
69+
partialSecretCache,
70+
&metav1.PartialObjectMetadata{
71+
TypeMeta: metav1.TypeMeta{
72+
Kind: "Secret",
73+
APIVersion: "v1",
74+
},
75+
},
76+
handler.TypedEnqueueRequestsFromMapFunc(
77+
r.secretToExtensionConfig,
78+
),
79+
)).
6980
WithOptions(options).
7081
WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)).
7182
Complete(r)
@@ -181,7 +192,7 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, extensionConfig *runti
181192

182193
// secretToExtensionConfig maps a secret to ExtensionConfigs with the corresponding InjectCAFromSecretAnnotation
183194
// to reconcile them on updates of the secrets.
184-
func (r *Reconciler) secretToExtensionConfig(ctx context.Context, secret client.Object) []reconcile.Request {
195+
func (r *Reconciler) secretToExtensionConfig(ctx context.Context, secret *metav1.PartialObjectMetadata) []reconcile.Request {
185196
result := []ctrl.Request{}
186197

187198
extensionConfigs := runtimev1.ExtensionConfigList{}

0 commit comments

Comments
 (0)