Skip to content

feat(evalhub): make EvalHub a cluster-scoped singleton deployed in operator namespace #655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ruivieira wants to merge 1 commit into
base: dev/evalhub-multi-tenancy
Choose a base branch
from
Open

feat(evalhub): make EvalHub a cluster-scoped singleton deployed in operator namespace #655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/evalhub/v1alpha1/evalhub_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
// EvalHub is the Schema for the evalhubs API
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'evalhub'",message="EvalHub resource name must be 'evalhub'"
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready`
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
Expand Down
3 changes: 1 addition & 2 deletions config/samples/evalhub_v1alpha1_evalhub.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
apiVersion: trustyai.opendatahub.io/v1alpha1
kind: EvalHub
metadata:
name: evalhub-sample
namespace: default
name: evalhub
spec:
# Number of replicas for the eval-hub deployment
replicas: 2
Expand Down
10 changes: 5 additions & 5 deletions controllers/evalhub/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (r *EvalHubReconciler) reconcileConfigMap(ctx context.Context, instance *ev
configMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: instance.Name + "-config",
Namespace: instance.Namespace,
Namespace: r.targetNamespace(),
},
}

Expand Down Expand Up @@ -268,7 +268,7 @@ func (r *EvalHubReconciler) reconcileProxyConfigMap(ctx context.Context, instanc
configMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: instance.Name + "-proxy-config",
Namespace: instance.Namespace,
Namespace: r.targetNamespace(),
},
}

Expand Down Expand Up @@ -305,7 +305,7 @@ func (r *EvalHubReconciler) generateProxyConfigData(instance *evalhubv1alpha1.Ev
proxyConfig := map[string]interface{}{
"authorization": map[string]interface{}{
"resourceAttributes": map[string]interface{}{
"namespace": instance.Namespace,
"namespace": r.targetNamespace(),
"apiGroup": "trustyai.opendatahub.io",
"resource": "evalhubs",
// kube-rbac-proxy expects the Kubernetes ResourceAttributes key "name".
Expand Down Expand Up @@ -390,7 +390,7 @@ func (r *EvalHubReconciler) reconcileProviderConfigMaps(ctx context.Context, ins
configMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: targetName,
Namespace: instance.Namespace,
Namespace: r.targetNamespace(),
},
}

Expand Down Expand Up @@ -450,7 +450,7 @@ func (r *EvalHubReconciler) reconcileServiceCAConfigMap(ctx context.Context, ins
configMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: instance.Name + "-service-ca",
Namespace: instance.Namespace,
Namespace: r.targetNamespace(),
Annotations: map[string]string{
// This annotation triggers OpenShift service CA operator to inject the CA certificate
"service.beta.openshift.io/inject-cabundle": "true",
Expand Down
8 changes: 4 additions & 4 deletions controllers/evalhub/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func (r *EvalHubReconciler) reconcileDeployment(ctx context.Context, instance *e
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: instance.Name,
Namespace: instance.Namespace,
Namespace: r.targetNamespace(),
},
}

Expand Down Expand Up @@ -109,7 +109,7 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
},
{
Name: "SERVICE_URL",
Value: fmt.Sprintf("https://%s.%s.svc.cluster.local:8443", instance.Name, instance.Namespace),
Value: fmt.Sprintf("https://%s.%s.svc.cluster.local:8443", instance.Name, r.targetNamespace()),
},
{
Name: "EVALHUB_INSTANCE_NAME",
Expand All @@ -121,7 +121,7 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
},
{
Name: "MLFLOW_WORKSPACE",
Value: instance.Namespace,
Value: r.targetNamespace(),
},
{
Name: "MLFLOW_TOKEN_PATH",
Expand Down Expand Up @@ -233,7 +233,7 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e

log.FromContext(ctx).Info("Configuring kube-rbac-proxy sidecar",
"evalHub", instance.Name,
"namespace", instance.Namespace,
"namespace", r.targetNamespace(),
"proxyImage", kubeRBACProxyImage)

// kube-rbac-proxy sidecar container
Expand Down
21 changes: 14 additions & 7 deletions controllers/evalhub/evalhub_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func (r *EvalHubReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return RequeueWithError(err)
}

log.Info("Reconciling EvalHub", "name", instance.Name, "namespace", instance.Namespace)
log.Info("Reconciling EvalHub", "name", instance.Name, "targetNamespace", r.targetNamespace())

// Handle deletion first to avoid blocking removal with status init.
if instance.DeletionTimestamp != nil {
Expand Down Expand Up @@ -220,6 +220,13 @@ func Requeue() (ctrl.Result, error) {
return ctrl.Result{Requeue: true}, nil
}

// targetNamespace returns the namespace where all EvalHub resources are deployed.
// EvalHub is a cluster-scoped singleton, so all its namespace-scoped resources
// (Deployments, Services, ConfigMaps, etc.) are created in the operator namespace.
func (r *EvalHubReconciler) targetNamespace() string {
return r.Namespace
}

// handleDeletion handles the deletion of EvalHub resources
func (r *EvalHubReconciler) handleDeletion(ctx context.Context, instance *evalhubv1alpha1.EvalHub) (ctrl.Result, error) {
log := log.FromContext(ctx)
Expand All @@ -244,8 +251,8 @@ func (r *EvalHubReconciler) handleDeletion(ctx context.Context, instance *evalhu
// cleanupClusterRoleBinding deletes EvalHub cluster-scoped RBAC resources upon instance deletion.
// Namespace-scoped resources (RoleBindings, ServiceAccounts) are garbage-collected via owner references.
func (r *EvalHubReconciler) cleanupClusterRoleBinding(ctx context.Context, instance *evalhubv1alpha1.EvalHub) error {
// Delete auth reviewer ClusterRoleBinding (cannot be owner-ref'd to a namespaced resource)
authCRBName := generateAuthReviewerClusterRoleBindingName(instance)
// Delete auth reviewer ClusterRoleBinding (cannot be owner-ref'd to a namespace-scoped resource)
authCRBName := generateAuthReviewerClusterRoleBindingName(instance, r.targetNamespace())
if err := r.deleteClusterRoleBinding(ctx, authCRBName); err != nil {
return err
}
Expand Down Expand Up @@ -278,10 +285,10 @@ func (r *EvalHubReconciler) deleteClusterRoleBinding(ctx context.Context, crbNam
func (r *EvalHubReconciler) updateStatus(ctx context.Context, instance *evalhubv1alpha1.EvalHub) error {
log := log.FromContext(ctx)

// Get the deployment
// Get the deployment from the operator namespace (cluster-scoped CR deploys here)
deployment := &appsv1.Deployment{}
err := r.Get(ctx, client.ObjectKey{
Namespace: instance.Namespace,
Namespace: r.targetNamespace(),
Name: instance.Name,
}, deployment)
if err != nil {
Expand All @@ -307,9 +314,9 @@ func (r *EvalHubReconciler) updateStatus(ctx context.Context, instance *evalhubv
instance.Status.Ready = corev1.ConditionTrue
instance.SetStatus("Ready", "DeploymentReady", "All replicas are ready", corev1.ConditionTrue)

// Set URL based on service (kube-rbac-proxy)
// Set URL based on service (kube-rbac-proxy) in the operator namespace
instance.Status.URL = fmt.Sprintf("https://%s.%s.svc.cluster.local:%d",
instance.Name, instance.Namespace, kubeRBACProxyPort)
instance.Name, r.targetNamespace(), kubeRBACProxyPort)
} else {
instance.Status.Phase = "Pending"
instance.Status.Ready = corev1.ConditionFalse
Expand Down
4 changes: 2 additions & 2 deletions controllers/evalhub/label_normalization_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func TestAuthReviewerCRB_AppNameLabelIsNormalizedWhenBindingNameTooLong(t *testi

require.NoError(t, reconciler.createServiceAccount(ctx, evalHub))

crbName := generateAuthReviewerClusterRoleBindingName(evalHub)
crbName := generateAuthReviewerClusterRoleBindingName(evalHub, ns)
require.Greater(t, len(crbName), 63, "test must exercise the >63 char case")

crb := &rbacv1.ClusterRoleBinding{}
Expand All @@ -71,5 +71,5 @@ func TestAuthReviewerCRB_AppNameLabelIsNormalizedWhenBindingNameTooLong(t *testi
lbl := crb.Labels["app.kubernetes.io/name"]
require.LessOrEqual(t, len(lbl), 63)
require.Regexp(t, dns1123LabelRe, lbl)
require.Equal(t, generateAuthReviewerClusterRoleBindingAppNameLabelValue(evalHub), lbl)
require.Equal(t, generateAuthReviewerClusterRoleBindingAppNameLabelValue(evalHub, ns), lbl)
}
Loading
Loading