feat(RHOAIENG-51619): add infrastructure part-of label for cache (#3231)

Author: davidebianchi

cmd/cloudmanager/app/cache.go

@@ -0,0 +1,57 @@
+package app
+
+import (
+	"fmt"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	k8slabels "k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/cloudmanager/common"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
+)
+
+// DefaultCacheOptions builds cache.Options for the given scheme, watching the default
+// managed namespaces shared by all cloud managers and filtering cluster-scoped
+// resources by the infrastructure part-of label.
+func DefaultCacheOptions(scheme *runtime.Scheme, kind string) (cache.Options, error) {
+	infraPartOfValue := labels.NormalizePartOfValue(kind)
+	if infraPartOfValue == "" {
+		return cache.Options{}, fmt.Errorf("infraPartOfValue must not be empty for label %s", labels.InfrastructurePartOf)
+	}
+
+	nsConfig := make(map[string]cache.Config, len(common.ManagedNamespaces()))
+	for _, ns := range common.ManagedNamespaces() {
+		nsConfig[ns] = cache.Config{}
+	}
+
+	labelSelector := k8slabels.Set{
+		labels.InfrastructurePartOf: infraPartOfValue,
+	}.AsSelector()
+
+	clusterScopedConfig := cache.ByObject{
+		Label: labelSelector,
+	}
+
+	return cache.Options{
+		Scheme:            scheme,
+		DefaultNamespaces: nsConfig,
+		ByObject: map[client.Object]cache.ByObject{
+			&rbacv1.ClusterRole{}:             clusterScopedConfig,
+			&rbacv1.ClusterRoleBinding{}:      clusterScopedConfig,
+			&extv1.CustomResourceDefinition{}: clusterScopedConfig,
+		},
+		DefaultTransform: func(in any) (any, error) {
+			// Nilcheck managed fields to avoid hitting https://github.com/kubernetes/kubernetes/issues/124337
+			if obj, err := meta.Accessor(in); err == nil && obj.GetManagedFields() != nil {
+				obj.SetManagedFields(nil)
+			}
+
+			return in, nil
+		},
+	}, nil
+}

cmd/cloudmanager/app/cache_test.go

@@ -0,0 +1,95 @@
+package app_test
+
+import (
+	"testing"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8slabels "k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	"github.com/opendatahub-io/opendatahub-operator/v2/cmd/cloudmanager/app"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestDefaultCacheOptions(t *testing.T) {
+	g := NewWithT(t)
+	s := runtime.NewScheme()
+
+	t.Run("label selector matches correct value", func(t *testing.T) {
+		g := NewWithT(t)
+
+		opts, err := app.DefaultCacheOptions(s, "azurekubernetesengine")
+		g.Expect(err).ShouldNot(HaveOccurred())
+
+		for obj, byObj := range opts.ByObject {
+			matching := k8slabels.Set{
+				labels.InfrastructurePartOf: "azurekubernetesengine",
+			}
+			g.Expect(byObj.Label.Matches(matching)).To(BeTrue(),
+				"label selector for %T should match %v", obj, matching)
+
+			nonMatching := k8slabels.Set{
+				labels.InfrastructurePartOf: "coreweavekubernetesengine",
+			}
+			g.Expect(byObj.Label.Matches(nonMatching)).To(BeFalse(),
+				"label selector for %T should not match %v", obj, nonMatching)
+		}
+	})
+
+	t.Run("normalizes infraPartOfValue to lowercase and trimmed", func(t *testing.T) {
+		g := NewWithT(t)
+
+		opts, err := app.DefaultCacheOptions(s, "  AzureKubernetesEngine  ")
+		g.Expect(err).ShouldNot(HaveOccurred())
+
+		for obj, byObj := range opts.ByObject {
+			matching := k8slabels.Set{
+				labels.InfrastructurePartOf: "azurekubernetesengine",
+			}
+			g.Expect(byObj.Label.Matches(matching)).To(BeTrue(),
+				"label selector for %T should match normalized value %v", obj, matching)
+		}
+	})
+
+	t.Run("returns error on empty infraPartOfValue", func(t *testing.T) {
+		g := NewWithT(t)
+
+		_, err := app.DefaultCacheOptions(s, "")
+		g.Expect(err).Should(HaveOccurred())
+	})
+
+	t.Run("returns error on whitespace-only infraPartOfValue", func(t *testing.T) {
+		g := NewWithT(t)
+
+		_, err := app.DefaultCacheOptions(s, "   ")
+		g.Expect(err).Should(HaveOccurred())
+	})
+
+	t.Run("uses provided scheme", func(t *testing.T) {
+		opts, err := app.DefaultCacheOptions(s, "azurekubernetesengine")
+		g.Expect(err).ShouldNot(HaveOccurred())
+
+		g.Expect(opts.Scheme).To(Equal(s))
+	})
+
+	t.Run("DefaultTransform clears ManagedFields", func(t *testing.T) {
+		g := NewWithT(t)
+
+		opts, err := app.DefaultCacheOptions(s, "azurekubernetesengine")
+		g.Expect(err).ShouldNot(HaveOccurred())
+		g.Expect(opts.DefaultTransform).ShouldNot(BeNil())
+
+		obj := &rbacv1.ClusterRole{}
+		obj.SetManagedFields([]metav1.ManagedFieldsEntry{{Manager: "test"}})
+
+		result, err := opts.DefaultTransform(obj)
+		g.Expect(err).ShouldNot(HaveOccurred())
+
+		transformed, ok := result.(*rbacv1.ClusterRole)
+		g.Expect(ok).To(BeTrue())
+		g.Expect(transformed.GetManagedFields()).To(BeNil())
+	})
+}

cmd/cloudmanager/app/provider.go

@@ -21,7 +21,7 @@ type Provider struct {
 	// NewReconciler creates and registers the provider's controller with the manager.
 	NewReconciler func(ctx context.Context, mgr ctrl.Manager) error
 	// CacheOptions returns the provider-specific cache configuration.
-	CacheOptions func(scheme *runtime.Scheme) cache.Options
+	CacheOptions func(scheme *runtime.Scheme) (cache.Options, error)
 	// ClientOptions returns the provider-specific client configuration.
 	// It always sets the unstructured cache to true.
 	ClientOptions func() client.Options

cmd/cloudmanager/app/run.go

@@ -4,14 +4,17 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
-	"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/cloudmanager/common"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/logger"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/manager"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/operatorconfig"
@@ -35,7 +38,7 @@ func Run(_ *cobra.Command, provider Provider) error {
 		return fmt.Errorf("invalid provider configuration: %w", err)
 	}
 
-	scheme := common.NewScheme(provider.AddToScheme)
+	scheme := newScheme(provider.AddToScheme)
 
 	clientOptions := provider.ClientOptions()
 	if clientOptions.Cache == nil {
@@ -44,6 +47,11 @@ func Run(_ *cobra.Command, provider Provider) error {
 	// The unstructured cache must be used.
 	clientOptions.Cache.Unstructured = true
 
+	cacheOptions, err := provider.CacheOptions(scheme)
+	if err != nil {
+		return fmt.Errorf("unable to get cache options: %w", err)
+	}
+
 	mgrOpts := ctrl.Options{
 		Scheme: scheme,
 		Metrics: ctrlmetrics.Options{
@@ -55,7 +63,7 @@ func Run(_ *cobra.Command, provider Provider) error {
 		HealthProbeBindAddress: cfg.HealthProbeAddr,
 		LeaderElection:         cfg.LeaderElection,
 		LeaderElectionID:       provider.LeaderElectionID,
-		Cache:                  provider.CacheOptions(scheme),
+		Cache:                  cacheOptions,
 		Client:                 clientOptions,
 	}
 
@@ -86,3 +94,16 @@ func Run(_ *cobra.Command, provider Provider) error {
 
 	return nil
 }
+
+func newScheme(addToSchemes ...func(*runtime.Scheme) error) *runtime.Scheme {
+	scheme := runtime.NewScheme()
+
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(apiextensionsv1.AddToScheme(scheme))
+
+	for _, addToScheme := range addToSchemes {
+		utilruntime.Must(addToScheme(scheme))
+	}
+
+	return scheme
+}

cmd/cloudmanager/azure/cmd.go

@@ -8,7 +8,6 @@ import (
 	ccmv1alpha1 "github.com/opendatahub-io/opendatahub-operator/v2/api/cloudmanager/azure/v1alpha1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/cmd/cloudmanager/app"
 	azurectrl "github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/cloudmanager/azure"
-	"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/cloudmanager/common"
 )
 
 // NewCmd returns the cobra command for the Azure cloud manager.
@@ -31,7 +30,7 @@ func NewCmd() *cobra.Command {
 	return cmd
 }
 
-func cacheOptions(scheme *runtime.Scheme) cache.Options {
-	defaultCacheOptions := common.DefaultCacheOptions(scheme)
-	return defaultCacheOptions
+func cacheOptions(scheme *runtime.Scheme) (cache.Options, error) {
+	kind := ccmv1alpha1.AzureKubernetesEngineKind
+	return app.DefaultCacheOptions(scheme, kind)
 }

cmd/cloudmanager/coreweave/cmd.go

@@ -7,7 +7,6 @@ import (
 
 	ccmv1alpha1 "github.com/opendatahub-io/opendatahub-operator/v2/api/cloudmanager/coreweave/v1alpha1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/cmd/cloudmanager/app"
-	"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/cloudmanager/common"
 	coreweavectrl "github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/cloudmanager/coreweave"
 )
 
@@ -31,7 +30,7 @@ func NewCmd() *cobra.Command {
 	return cmd
 }
 
-func cacheOptions(scheme *runtime.Scheme) cache.Options {
-	defaultCacheOptions := common.DefaultCacheOptions(scheme)
-	return defaultCacheOptions
+func cacheOptions(scheme *runtime.Scheme) (cache.Options, error) {
+	kind := ccmv1alpha1.CoreWeaveKubernetesEngineKind
+	return app.DefaultCacheOptions(scheme, kind)
 }

internal/controller/cloudmanager/azure/azurekubernetesengine_controller_test.go

@@ -9,6 +9,7 @@ import (
 	ccmv1alpha1 "github.com/opendatahub-io/opendatahub-operator/v2/api/cloudmanager/azure/v1alpha1"
 	ccmcommon "github.com/opendatahub-io/opendatahub-operator/v2/api/cloudmanager/common"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/utils/test/matchers/jq"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/utils/test/testf"
 
@@ -45,6 +46,26 @@ func TestAzureKubernetesEngine(t *testing.T) {
 			Name: "servicemesh-operator3", Namespace: "istio-system",
 		}).Eventually().Should(Not(BeNil()))
 	})
+
+	t.Run("sets infrastructure label on deployed resources", func(t *testing.T) {
+		wt := tc.NewWithT(t)
+
+		createAzureCR(t, wt, ccmcommon.Dependencies{
+			CertManager: ccmcommon.CertManagerDependency{ManagementPolicy: ccmcommon.Managed},
+		})
+
+		nn := types.NamespacedName{Name: ccmv1alpha1.AzureKubernetesEngineInstanceName}
+
+		wt.Get(gvk.AzureKubernetesEngine, nn).Eventually().Should(
+			jq.Match(`.status.conditions[] | select(.type == "Ready") | .status == "True"`),
+		)
+
+		wt.Get(gvk.Deployment, types.NamespacedName{
+			Name: "cert-manager-operator-controller-manager", Namespace: "cert-manager-operator",
+		}).Eventually().Should(
+			jq.Match(`.metadata.labels."%s" == "azurekubernetesengine"`, labels.InfrastructurePartOf),
+		)
+	})
 }
 
 func createAzureCR(t *testing.T, wt *testf.WithT, deps ccmcommon.Dependencies) {