[Spark] Support spark-operator on multi-namespace deployments (#258)

Co-authored-by: Alex Toker <alext@mckinsey.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: 3 people

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.11.0-rc.12
+version: 0.11.0-rc.13
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/admin_installation_values.yaml

@@ -40,6 +40,26 @@ seaweedfs:
   enabled: false
 
 spark-operator:
+  enabled: true
+  fullnameOverride: spark-operator
+  controller:
+    replicas: 0           # No running pods in admin
+    rbac:
+      create: true        # Creates ClusterRole (shared by all user namespaces)
+    serviceAccount:
+      create: true
+  webhook:
+    enable: true
+    replicas: 1
+  spark:
+    jobNamespaces:
+      - ""              # All namespaces (no namespaceSelector on webhook)
+    serviceAccount:
+      create: false       # No sparkapp SA in admin
+    rbac:
+      create: false
+
+spark:
   enabled: false
 
 pipelines:

charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml

@@ -44,7 +44,29 @@ timescaledb:
     nodePort: ""
 
 spark-operator:
-  enabled: false
+  enabled: true
+  fullnameOverride: spark-operator
+  controller:
+    replicas: 1
+    rbac:
+      create: false
+    serviceAccount:
+      create: true
+    leaderElection:
+      enable: true
+  webhook:
+    enable: false
+  spark:
+    jobNamespaces:
+      - mlrun
+    serviceAccount:
+      create: true
+      name: sparkapp
+    rbac:
+      create: true
+
+spark:
+  enabled: true
 
 pipelines:
   service:

charts/mlrun-ce/non_admin_installation_values.yaml

@@ -38,7 +38,29 @@ seaweedfs:
   enabled: true
 
 spark-operator:
-  enabled: false
+  enabled: true
+  fullnameOverride: spark-operator
+  controller:
+    replicas: 1           # Controller runs in user namespace
+    rbac:
+      create: false       # ClusterRole already exists from admin
+    serviceAccount:
+      create: true
+    leaderElection:
+      enable: true
+  webhook:
+    enable: false
+  spark:
+    jobNamespaces:
+      - mlrun             # Override with actual namespace at install time
+    serviceAccount:
+      create: true
+      name: sparkapp
+    rbac:
+      create: true        # Creates sparkapp Role + RoleBinding
+
+spark:
+  enabled: true
 
 pipelines:
   service:

charts/mlrun-ce/templates/config/mlrun-spark-config.yaml

@@ -1,4 +1,4 @@
-{{- if index .Values "spark-operator" "enabled" -}}
+{{- if .Values.spark.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:

charts/mlrun-ce/templates/spark-operator/spark-controller-rbac.yaml

@@ -0,0 +1,68 @@
+{{- $sparkOp := index .Values "spark-operator" -}}
+{{- $rbacCreate := true -}}
+{{- if hasKey $sparkOp "controller" -}}
+  {{- if hasKey $sparkOp.controller "rbac" -}}
+    {{- $rbacCreate = $sparkOp.controller.rbac.create -}}
+  {{- end -}}
+{{- end -}}
+{{- if and $sparkOp.enabled (not $rbacCreate) -}}
+{{- /*
+  This template renders only in user multi-NS mode:
+  - spark-operator subchart is enabled (controller Deployment runs here)
+  - controller.rbac.create is false (ClusterRole already exists from admin namespace)
+
+  It creates:
+  1. RoleBinding: controller SA → shared ClusterRole (namespace-scoped access)
+  2. Role + RoleBinding: leader election leases (coordination.k8s.io)
+*/ -}}
+---
+# RoleBinding: Grant controller SA access to the shared ClusterRole (namespace-scoped)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: spark-operator-controller
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: spark-controller-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: spark-operator-controller
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: spark-operator-controller
+  apiGroup: rbac.authorization.k8s.io
+---
+# Role: Leader election leases
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: spark-operator-controller-leases
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: spark-controller-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["create", "get", "update"]
+---
+# RoleBinding: Grant controller SA access to leader election leases
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: spark-operator-controller-leases
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: spark-controller-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: spark-operator-controller
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: spark-operator-controller-leases
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/mlrun-ce/values.yaml

@@ -645,3 +645,11 @@ kafka:
     # Empty means "use the release namespace"
     # Example: "controller" if that's where you installed the operator
     operatorNamespace: ""
+
+# Spark configuration for multi-NS deployments
+# Controls CE-level spark resources (mlrun-spark-config ConfigMap)
+# In single-NS mode, both spark.enabled and spark-operator.enabled are true
+# In multi-NS admin mode, spark.enabled is false (no ConfigMap needed)
+# In multi-NS user mode, spark.enabled is true (ConfigMap needed for MLRun)
+spark:
+  enabled: true