add: mutating webhook

VanillaSpoon · astefanutti · commit 3fa247f1c0c0 · 2024-04-17T18:31:23.000+02:00
diff --git a/PROJECT b/PROJECT
@@ -15,5 +15,9 @@ resources:
   domain: codeflare.dev
   group: ray
   kind: RayCluster
+  path: github.com/project-codeflare/codeflare-operator/pkg/controllers
   version: v1
+  webhooks:
+    defaulting: true
+    webhookVersion: v1
 version: "3"
diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml
@@ -0,0 +1,3 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml
@@ -0,0 +1,5 @@
+resources:
+- certificate.yaml
+
+configurations:
+- kustomizeconfig.yaml
diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml
@@ -0,0 +1,16 @@
+# This configuration is for teaching kustomize how to update name ref and var substitution
+nameReference:
+- kind: Issuer
+  group: cert-manager.io
+  fieldSpecs:
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/issuerRef/name
+
+varReference:
+- kind: Certificate
+  group: cert-manager.io
+  path: spec/commonName
+- kind: Certificate
+  group: cert-manager.io
+  path: spec/dnsNames
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -14,10 +14,15 @@ commonLabels:
   app.kubernetes.io/part-of: codeflare
 
 bases:
-- ../rbac
-- ../manager
+  - ../rbac
+  - ../manager
+  - ../webhook
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 # - ../prometheus
 
 resources:
-- metrics_service.yaml
+  - metrics_service.yaml
+
+patches:
+- path: manager_webhook_patch.yaml
+- path: webhookcainjection_patch.yaml
diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml
@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: codeflare-operator-raycluster-webhook-cert
diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml
@@ -0,0 +1,15 @@
+# This patch add annotation to admission webhook config
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: mutatingwebhookconfiguration
+    app.kubernetes.io/instance: mutating-webhook-configuration
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: codeflare-operator
+    app.kubernetes.io/part-of: codeflare-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: mutating-webhook-configuration
+  annotations:
+     service.beta.openshift.io/inject-cabundle: "true"
+     service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert
diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml
@@ -0,0 +1,6 @@
+resources:
+- manifests.yaml
+- service.yaml
+
+configurations:
+- kustomizeconfig.yaml
diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml
@@ -0,0 +1,25 @@
+# the following config is for teaching kustomize where to look at when substituting vars.
+# It requires kustomize v2.1.0 or newer to work properly.
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: MutatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+  - kind: ValidatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+
+namespace:
+- kind: MutatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+- kind: ValidatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+
+varReference:
+- path: metadata/annotations
diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-ray-io-v1-raycluster
+  failurePolicy: Fail
+  name: mraycluster.kb.io
+  rules:
+  - apiGroups:
+    - ray.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - rayclusters
+  sideEffects: None
diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: service
+    app.kubernetes.io/part-of: codeflare-operator 
+    app.kubernetes.io/instance: webhook-service
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: codeflare-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: webhook-service
+  namespace: openshift-operators
+  annotations:
+       service.beta.openshift.io/serving-cert-secret-name:  codeflare-operator-raycluster-webhook-cert
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 9443
+  selector:
+    app.kubernetes.io/part-of: codeflare 
+    app.kubernetes.io/name: codeflare-operator
diff --git a/main.go b/main.go
@@ -147,6 +147,9 @@ func main() {
 	})
 	exitOnError(err, "unable to start manager")
 
+	rayClusterDefaulter := &controllers.RayClusterDefaulter{}
+	exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook")
+
 	ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
 	if ok {
 		rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay}
diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go
@@ -0,0 +1,122 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+// log is for logging in this package.
+var rayclusterlog = logf.Log.WithName("raycluster-resource")
+
+func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(&rayv1.RayCluster{}).
+		WithDefaulter(r).
+		Complete()
+}
+
+//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1
+
+type RayClusterDefaulter struct{}
+
+var _ webhook.CustomDefaulter = &RayClusterDefaulter{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error {
+	raycluster := obj.(*rayv1.RayCluster)
+
+	rayclusterlog.Info("default", "name", raycluster.Name)
+	// Check and add OAuth proxy if it does not exist.
+	alreadyExists := false
+	for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
+		if container.Name == "oauth-proxy" {
+			rayclusterlog.Info("OAuth sidecar already exists, no patch needed")
+			alreadyExists = true
+			break // exits the for loop
+		}
+	}
+
+	if !alreadyExists {
+		rayclusterlog.Info("Adding OAuth sidecar container")
+		// definition of the new container
+		newOAuthSidecar := corev1.Container{
+			Name:  "oauth-proxy",
+			Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366",
+			Ports: []corev1.ContainerPort{
+				{ContainerPort: 8443, Name: "oauth-proxy"},
+			},
+			Args: []string{
+				"--https-address=:8443",
+				"--provider=openshift",
+				"--openshift-service-account=" + raycluster.Name + "-oauth-proxy",
+				"--upstream=http://localhost:8265",
+				"--tls-cert=/etc/tls/private/tls.crt",
+				"--tls-key=/etc/tls/private/tls.key",
+				"--cookie-secret=$(COOKIE_SECRET)",
+				"--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}",
+				"--added-label=True",
+			},
+			Env: []corev1.EnvVar{
+				{
+					Name: "COOKIE_SECRET",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: raycluster.Name + "-oauth-config",
+							},
+							Key: "cookie_secret",
+						},
+					},
+				},
+			},
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      "proxy-tls-secret",
+					MountPath: "/etc/tls/private",
+					ReadOnly:  true,
+				},
+			},
+		}
+
+		// Adding the new OAuth sidecar container
+		raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar)
+
+		tlsSecretVolume := corev1.Volume{
+			Name: "proxy-tls-secret",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: raycluster.Name + "-proxy-tls-secret",
+				},
+			},
+		}
+
+		raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume)
+
+		// Ensure the service account is set
+		if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" {
+			raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy"
+		}
+	}
+	return nil
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# The following manifests contain a self-signed issuer CR and a certificate CR.`
	`2`	`+# More document can be found at https://docs.cert-manager.io`
	`3`	`+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.`