Add ingressDomain to configmap to work with local and workflow e2e tests

Author: ChristianZaccaria

config/e2e/config.yaml

@@ -6,3 +6,4 @@ data:
   config.yaml: |
     kuberay:
       rayDashboardOAuthEnabled: false
+      ingressDomain: "kind"

main.go

@@ -115,6 +115,7 @@ func main() {
 		},
 		KubeRay: &config.KubeRayConfiguration{
 			RayDashboardOAuthEnabled: pointer.Bool(true),
+			IngressDomain:            "fake.domain",
 		},
 	}
 

pkg/config/config.go

@@ -33,6 +33,8 @@ type CodeFlareOperatorConfiguration struct {
 
 type KubeRayConfiguration struct {
 	RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"`
+
+	IngressDomain string `json:"ingressDomain,omitempty"`
 }
 
 type ControllerManager struct {

pkg/controllers/raycluster_controller.go

@@ -184,21 +184,21 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		}
 
 	} else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift {
-		ingressDomain := "" // TODO: ingressDomain should be retrieved by the CFO and used here to fix local_interactive. Jira: https://issues.redhat.com/browse/RHOAIENG-5330
+		logger.Info("We detected being on Vanilla Kubernetes!")
 		logger.Info("Creating Dashboard Ingress")
-		_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
+		dashboardName := dashboardNameFromCluster(&cluster)
+		_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
 		if err != nil {
 			// This log is info level since errors are not fatal and are expected
 			logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true)
 			return ctrl.Result{RequeueAfter: requeueTime}, err
 		}
-		if ingressDomain != "" {
-			logger.Info("Creating RayClient Ingress")
-			_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
-			if err != nil {
-				logger.Error(err, "Failed to update RayClient Ingress")
-				return ctrl.Result{RequeueAfter: requeueTime}, err
-			}
+		logger.Info("Creating RayClient Ingress")
+		rayClientName := rayClientNameFromCluster(&cluster)
+		_, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
+		if err != nil {
+			logger.Error(err, "Failed to update RayClient Ingress")
+			return ctrl.Result{RequeueAfter: requeueTime}, err
 		}
 	}
 

pkg/controllers/support.go

@@ -8,7 +8,6 @@ import (
 	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
 
 	networkingv1 "k8s.io/api/networking/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	v1 "k8s.io/client-go/applyconfigurations/meta/v1"
@@ -39,7 +38,7 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf
 }
 
 // Create an Ingress object for the RayCluster
-func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *networkingv1ac.IngressApplyConfiguration {
+func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
 	return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace).
 		WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
 		WithAnnotations(map[string]string{
@@ -55,7 +54,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *n
 		WithSpec(networkingv1ac.IngressSpec().
 			WithIngressClassName("nginx").
 			WithRules(networkingv1ac.IngressRule().
-				WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingressDomain).
+				WithHost(ingressHost).
 				WithHTTP(networkingv1ac.HTTPIngressRuleValue().
 					WithPaths(networkingv1ac.HTTPIngressPath().
 						WithPath("/").
@@ -85,7 +84,7 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo
 			WithUID(types.UID(cluster.UID))).
 		WithSpec(networkingv1ac.IngressSpec().
 			WithRules(networkingv1ac.IngressRule().
-				WithHost(ingressHost). // KinD hostname or ingressDomain
+				WithHost(ingressHost). // Full Hostname
 				WithHTTP(networkingv1ac.HTTPIngressRuleValue().
 					WithPaths(networkingv1ac.HTTPIngressPath().
 						WithPath("/").
@@ -104,19 +103,6 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo
 		)
 }
 
-// isOnKindCluster checks if the current cluster is a KinD cluster.
-// It searches for a node with a label commonly used by KinD clusters.
-func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) {
-	nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
-		LabelSelector: "kubernetes.io/hostname=kind-control-plane",
-	})
-	if err != nil {
-		return false, err
-	}
-	// If we find one or more nodes with the label, assume it's a KinD cluster.
-	return len(nodes.Items) > 0, nil
-}
-
 // getDiscoveryClient returns a discovery client for the current reconciler
 func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) {
 	return discovery.NewDiscoveryClientForConfig(config)
@@ -153,15 +139,15 @@ func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster *
 }
 
 // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain.
-func getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) string {
-	logger := ctrl.LoggerFrom(ctx)
-	onKind, _ := isOnKindCluster(clientset)
-	if onKind && ingressDomain == "" {
-		logger.Info("We detected being on a KinD cluster!")
-		return "kind"
+func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) string {
+	ingressDomain := "fake.domain"
+	if r.Config != nil && r.Config.IngressDomain != "" {
+		ingressDomain = r.Config.IngressDomain
 	}
-	logger.Info("We detected being on Vanilla Kubernetes!")
-	return fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain)
+	if ingressDomain == "kind" {
+		return ingressDomain
+	}
+	return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain)
 }
 
 func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool {