add watches for owned resources and CRBs

Signed-off-by: Kevin <[email protected]>

Author: KPostOffice

config/rbac/role.yaml

@@ -126,6 +126,7 @@ rules:
   - delete
   - get
   - patch
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -136,6 +137,7 @@ rules:
   - get
   - patch
   - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -146,6 +148,7 @@ rules:
   - get
   - patch
   - update
+  - watch
 - apiGroups:
   - dscinitialization.opendatahub.io
   resources:
@@ -218,6 +221,7 @@ rules:
   - get
   - patch
   - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:
@@ -264,6 +268,7 @@ rules:
   - get
   - patch
   - update
+  - watch
 - apiGroups:
   - route.openshift.io
   resources:
@@ -275,6 +280,7 @@ rules:
   - get
   - patch
   - update
+  - watch
 - apiGroups:
   - scheduling.k8s.io
   resources:

pkg/controllers/raycluster_controller.go

@@ -31,6 +31,8 @@ import (
 	"time"
 
 	dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+
 	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
 
 	corev1 "k8s.io/api/core/v1"
@@ -48,6 +50,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	routev1 "github.com/openshift/api/route/v1"
 	routev1ac "github.com/openshift/client-go/route/applyconfigurations/route/v1"
@@ -88,12 +92,12 @@ var (
 // +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update
-// +kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=get;create;update;patch;delete
-// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get
-// +kubebuilder:rbac:groups=core,resources=services,verbs=get;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;create;update;patch;delete
-// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete
+// +kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=get;create;update;patch;delete;watch
+// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;create;update;patch;delete;watch
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get;watch
+// +kubebuilder:rbac:groups=core,resources=services,verbs=get;create;update;patch;delete;watch
+// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;create;update;patch;delete;watch
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete;watch
 // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create;
 // +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create;
 // +kubebuilder:rbac:groups=dscinitialization.opendatahub.io,resources=dscinitializations,verbs=get;list;watch
@@ -301,7 +305,7 @@ func crbNameFromCluster(cluster *rayv1.RayCluster) string {
 func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration {
 	return rbacv1ac.ClusterRoleBinding(
 		crbNameFromCluster(cluster)).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name, "ray.openshift.ai/cluster-namespace": cluster.Namespace}).
 		WithSubjects(
 			rbacv1ac.Subject().
 				WithKind("ServiceAccount").
@@ -322,7 +326,7 @@ func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string {
 
 func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountApplyConfiguration {
 	return corev1ac.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithAnnotations(map[string]string{
 			"serviceaccounts.openshift.io/oauth-redirectreference.first": "" +
 				`{"kind":"OAuthRedirectReference","apiVersion":"v1",` +
@@ -343,7 +347,7 @@ func rayClientNameFromCluster(cluster *rayv1.RayCluster) string {
 
 func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfiguration {
 	return routev1ac.Route(dashboardNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithSpec(routev1ac.RouteSpec().
 			WithTo(routev1ac.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))).
 			WithPort(routev1ac.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))).
@@ -367,7 +371,7 @@ func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string {
 
 func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfiguration {
 	return corev1ac.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithAnnotations(map[string]string{"service.beta.openshift.io/serving-cert-secret-name": oauthServiceTLSSecretName(cluster)}).
 		WithSpec(
 			corev1ac.ServiceSpec().
@@ -397,7 +401,7 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.
 	cookieSecret := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 
 	return corev1ac.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithStringData(map[string]string{"cookie_secret": cookieSecret}).
 		WithOwnerReferences(
 			metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion),
@@ -410,7 +414,7 @@ func caSecretNameFromCluster(cluster *rayv1.RayCluster) string {
 
 func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.SecretApplyConfiguration {
 	return corev1ac.Secret(caSecretNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithData(map[string][]byte{
 			CAPrivateKeyKey: key,
 			CACertKey:       cert,
@@ -466,7 +470,7 @@ func generateCACertificate() ([]byte, []byte, error) {
 }
 func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration {
 	return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithSpec(networkingv1ac.NetworkPolicySpec().
 			WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})).
 			WithIngress(
@@ -488,7 +492,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf
 		allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)))
 	}
 	return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithSpec(networkingv1ac.NetworkPolicySpec().
 			WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})).
 			WithIngress(
@@ -551,5 +555,27 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(controllerName).
 		For(&rayv1.RayCluster{}).
+		Owns(&corev1.ServiceAccount{}).
+		Owns(&corev1.Service{}).
+		Owns(&corev1.Secret{}).
+		Owns(&routev1.Route{}).
+		Owns(&networkingv1.Ingress{}).
+		Watches(&rbacv1.ClusterRoleBinding{}, handler.EnqueueRequestsFromMapFunc(
+			func(c context.Context, o client.Object) []reconcile.Request {
+				name, ok := o.GetLabels()["ray.openshift.ai/cluster-name"]
+				if !ok {
+					return []reconcile.Request{}
+				}
+				namespace, ok := o.GetLabels()["ray.openshift.ai/cluster-namespace"]
+				if !ok {
+					return []reconcile.Request{}
+				}
+				return []reconcile.Request{{
+					NamespacedName: client.ObjectKey{
+						Name:      name,
+						Namespace: namespace,
+					}}}
+			}),
+		).
 		Complete(r)
 }

pkg/controllers/support.go

@@ -20,7 +20,7 @@ func serviceNameFromCluster(cluster *rayv1.RayCluster) string {
 
 func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
 	return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithSpec(routeapply.RouteSpec().
 			WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)).
 			WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))).
@@ -33,7 +33,7 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf
 
 func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
 	return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithAnnotations(map[string]string{
 			"nginx.ingress.kubernetes.io/rewrite-target":  "/",
 			"nginx.ingress.kubernetes.io/ssl-redirect":    "true",
@@ -68,7 +68,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net
 
 func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
 	return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace).
-		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
+		WithLabels(map[string]string{"ray.openshift.ai/cluster-name": cluster.Name}).
 		WithOwnerReferences(v1.OwnerReference().
 			WithAPIVersion(cluster.APIVersion).
 			WithKind(cluster.Kind).