From 24fde7b0b8e46e4057c0fbd99d9e217706a97d57 Mon Sep 17 00:00:00 2001 From: pehlicd Date: Mon, 2 Feb 2026 12:18:11 +0100 Subject: [PATCH 1/3] feat: auto tolerate daemonsets with MAP Signed-off-by: pehlicd --- config/admission-policy/binding.yaml | 15 ++++ config/admission-policy/configmap.yaml | 12 +++ config/admission-policy/kustomization.yaml | 12 +++ config/admission-policy/policy.yaml | 71 ++++++++++++++++++ docs/admission-policy.md | 54 +++++++++++++ go.mod | 2 +- .../nodereadinessrule_controller.go | 75 +++++++++++++++++++ 7 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 config/admission-policy/binding.yaml create mode 100644 config/admission-policy/configmap.yaml create mode 100644 config/admission-policy/kustomization.yaml create mode 100644 config/admission-policy/policy.yaml create mode 100644 docs/admission-policy.md diff --git a/config/admission-policy/binding.yaml b/config/admission-policy/binding.yaml new file mode 100644 index 0000000..76d850a --- /dev/null +++ b/config/admission-policy/binding.yaml @@ -0,0 +1,15 @@ +--- +# MutatingAdmissionPolicyBinding binds the policy to the ConfigMap parameter +apiVersion: admissionregistration.k8s.io/v1alpha1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: inject-daemonset-readiness-tolerations-binding +spec: + policyName: inject-daemonset-readiness-tolerations + # Reference the ConfigMap containing toleration data + paramRef: + name: readiness-taints + namespace: nrr-system + parameterNotFoundAction: Deny + matchResources: + namespaceSelector: {} diff --git a/config/admission-policy/configmap.yaml b/config/admission-policy/configmap.yaml new file mode 100644 index 0000000..eab9101 --- /dev/null +++ b/config/admission-policy/configmap.yaml @@ -0,0 +1,12 @@ +--- +# ConfigMap that stores readiness tolerations +# This will be populated/updated by the NodeReadinessRule controller +apiVersion: v1 +kind: ConfigMap +metadata: + name: readiness-taints + namespace: nrr-system +data: + # Store each toleration key separately for easier CEL access + # Format: key1=readiness.k8s.io/NetworkReady,key2=readiness.k8s.io/StorageReady + taint-keys: "" diff --git a/config/admission-policy/kustomization.yaml b/config/admission-policy/kustomization.yaml new file mode 100644 index 0000000..b3e49bc --- /dev/null +++ b/config/admission-policy/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- configmap.yaml +- policy.yaml +- binding.yaml + +labels: +- pairs: + app.kubernetes.io/name: nrrcontroller + app.kubernetes.io/component: admission-policy diff --git a/config/admission-policy/policy.yaml b/config/admission-policy/policy.yaml new file mode 100644 index 0000000..78e934f --- /dev/null +++ b/config/admission-policy/policy.yaml @@ -0,0 +1,71 @@ +--- +# MutatingAdmissionPolicy for automatic DaemonSet toleration injection +# Reads taint keys from a ConfigMap parameter resource +# Requires: MutatingAdmissionPolicy feature enabled in the cluster +apiVersion: admissionregistration.k8s.io/v1alpha1 +kind: MutatingAdmissionPolicy +metadata: + name: inject-daemonset-readiness-tolerations +spec: + failurePolicy: Fail + + # Define what this policy watches + matchConstraints: + resourceRules: + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["daemonsets"] + + # Reference the ConfigMap that contains toleration data + paramKind: + apiVersion: v1 + kind: ConfigMap + + # Variables for CEL expressions + variables: + # Check if opt-out annotation is set + - name: optedOut + expression: | + has(object.metadata.annotations) && + object.metadata.annotations.exists(k, k == "readiness.k8s.io/auto-tolerate" && object.metadata.annotations[k] == "false") + + # Get existing tolerations (empty array if none) + - name: existingTolerations + expression: | + has(object.spec.template.spec.tolerations) ? + object.spec.template.spec.tolerations : [] + + # Get taint keys from ConfigMap and parse to array + - name: taintKeys + expression: | + ("taint-keys" in params.data) && params.data["taint-keys"] != "" ? + params.data["taint-keys"].split(",") : [] + + # Create tolerations from taint keys (as plain maps since CEL has issues with complex types) + - name: tolerationsToInject + expression: | + variables.taintKeys + .filter(key, !variables.existingTolerations.exists(t, t.key == key)) + .map(key, { + "key": key, + "operator": "Exists", + "effect": "NoSchedule" + }) + + # Apply mutations + mutations: + - patchType: JSONPatch + jsonPatch: + expression: | + !variables.optedOut && size(variables.tolerationsToInject) > 0 ? + [ + JSONPatch{ + op: has(object.spec.template.spec.tolerations) ? "replace" : "add", + path: "/spec/template/spec/tolerations", + value: variables.existingTolerations + variables.tolerationsToInject + } + ] : [] + + # Never reinvoke this policy + reinvocationPolicy: Never diff --git a/docs/admission-policy.md b/docs/admission-policy.md new file mode 100644 index 0000000..07bcada --- /dev/null +++ b/docs/admission-policy.md @@ -0,0 +1,54 @@ +# MutatingAdmissionPolicy for DaemonSet Toleration Injection + +This document describes how to deploy and use the MutatingAdmissionPolicy-based approach for automatically injecting readiness tolerations into DaemonSets. + +## Overview + +The MutatingAdmissionPolicy approach uses Kubernetes's native admission control mechanism with CEL (Common Expression Language) to inject tolerations **without running a webhook server**. This provides a simpler, more declarative alternative to the webhook-based approach. + +## Requirements + +> [!IMPORTANT] +> MutatingAdmissionPolicy is needed to be enabled in the cluster. + +- Feature gate: `MutatingAdmissionPolicy=true` +- Runtime config: `admissionregistration.k8s.io/v1alpha1=true` +- `kubectl` configured to access your cluster +- NodeReadinessRule CRDs installed + +## Architecture + +``` +User applies DaemonSet + ↓ +API Server evaluates CEL policy + ↓ +Fetches Tolerations ConfigMap which contains the tolerations to be injected + ↓ +Injects tolerations (if applicable) + ↓ +DaemonSet created with tolerations +``` + +## Deployment + +### Option 1: Using kustomize + +```bash +# Install CRDs first +make install + +# Deploy the admission policy +kubectl apply -k config/admission-policy +``` + +### Option 2: Direct kubectl apply + +```bash +# Install CRDs first +make install + +# Deploy policy and binding +kubectl apply -f config/admission-policy/policy.yaml +kubectl apply -f config/admission-policy/binding.yaml +``` diff --git a/go.mod b/go.mod index 01fca93..9e132dc 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( k8s.io/api v0.34.0 k8s.io/apimachinery v0.34.0 k8s.io/client-go v0.34.0 + k8s.io/klog/v2 v2.130.1 sigs.k8s.io/controller-runtime v0.22.1 ) @@ -91,7 +92,6 @@ require ( k8s.io/apiextensions-apiserver v0.34.0 // indirect k8s.io/apiserver v0.34.0 // indirect k8s.io/component-base v0.34.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect diff --git a/internal/controller/nodereadinessrule_controller.go b/internal/controller/nodereadinessrule_controller.go index 9bb2420..aea7417 100644 --- a/internal/controller/nodereadinessrule_controller.go +++ b/internal/controller/nodereadinessrule_controller.go @@ -166,6 +166,12 @@ func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{RequeueAfter: time.Minute}, err } + // Sync taints to ConfigMap for MutatingAdmissionPolicy + if err := r.Controller.syncTaintsConfigMap(ctx); err != nil { + log.Error(err, "Failed to sync taints configmap", "rule", rule.Name) + // Don't fail reconciliation for this - log and continue + } + return ctrl.Result{}, nil } @@ -671,6 +677,75 @@ func (r *RuleReadinessController) cleanupNodesAfterSelectorChange(ctx context.Co return nil } +// syncTaintsConfigMap synchronizes readiness taints to a ConfigMap for admission policy. +func (r *RuleReadinessController) syncTaintsConfigMap(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + + // List all NodeReadinessRules + var ruleList readinessv1alpha1.NodeReadinessRuleList + if err := r.List(ctx, &ruleList); err != nil { + return fmt.Errorf("failed to list NodeReadinessRules: %w", err) + } + + // Extract unique taint keys with readiness.k8s.io/ prefix and NoSchedule effect + taintKeysSet := make(map[string]struct{}) + for _, rule := range ruleList.Items { + if rule.Spec.Taint.Key != "" && + strings.HasPrefix(rule.Spec.Taint.Key, "readiness.k8s.io/") && + rule.Spec.Taint.Effect == corev1.TaintEffectNoSchedule { + taintKeysSet[rule.Spec.Taint.Key] = struct{}{} + } + } + + // Convert set to comma-separated string + taintKeys := make([]string, 0, len(taintKeysSet)) + for key := range taintKeysSet { + taintKeys = append(taintKeys, key) + } + taintKeysStr := strings.Join(taintKeys, ",") + + // Update or create ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "readiness-taints", + Namespace: "nrr-system", + }, + } + + // Try to get existing ConfigMap + existingCM := &corev1.ConfigMap{} + err := r.Get(ctx, client.ObjectKey{Name: "readiness-taints", Namespace: "nrr-system"}, existingCM) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get configmap: %w", err) + } + + // Set data + cm.Data = map[string]string{ + "taint-keys": taintKeysStr, + } + + if apierrors.IsNotFound(err) { + // Create new ConfigMap + log.Info("Creating readiness-taints ConfigMap", "taintCount", len(taintKeys)) + if err := r.Create(ctx, cm); err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + } else { + // Update existing ConfigMap + log.V(1).Info("Updating readiness-taints ConfigMap", "taintCount", len(taintKeys)) + patch := client.MergeFrom(existingCM) + existingCM.Data = cm.Data + if err := r.Patch(ctx, existingCM, patch); err != nil { + return fmt.Errorf("failed to update configmap: %w", err) + } + } + + log.V(2).Info("Successfully synced taints to ConfigMap", + "totalRules", len(ruleList.Items), + "readinessTaints", len(taintKeys)) + return nil +} + func (r *RuleReconciler) ensureFinalizer(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule, finalizer string) (finalizerAdded bool, err error) { // Finalizers can only be added when the deletionTimestamp is not set. if !rule.GetDeletionTimestamp().IsZero() { From e00f3e43d4224b9d632d6fe3720869549f126e12 Mon Sep 17 00:00:00 2001 From: pehlicd Date: Sun, 8 Feb 2026 17:04:57 +0100 Subject: [PATCH 2/3] feat: auto tolerate daemonsets with MAP Signed-off-by: pehlicd --- config/admission-policy/binding.yaml | 2 +- config/admission-policy/policy.yaml | 2 +- config/rbac/role.yaml | 1 + docs/admission-policy.md | 2 +- .../controller/nodereadinessrule_controller.go | 16 +++++++++++++++- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/config/admission-policy/binding.yaml b/config/admission-policy/binding.yaml index 76d850a..8107aca 100644 --- a/config/admission-policy/binding.yaml +++ b/config/admission-policy/binding.yaml @@ -1,6 +1,6 @@ --- # MutatingAdmissionPolicyBinding binds the policy to the ConfigMap parameter -apiVersion: admissionregistration.k8s.io/v1alpha1 +apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingAdmissionPolicyBinding metadata: name: inject-daemonset-readiness-tolerations-binding diff --git a/config/admission-policy/policy.yaml b/config/admission-policy/policy.yaml index 78e934f..6da22f1 100644 --- a/config/admission-policy/policy.yaml +++ b/config/admission-policy/policy.yaml @@ -2,7 +2,7 @@ # MutatingAdmissionPolicy for automatic DaemonSet toleration injection # Reads taint keys from a ConfigMap parameter resource # Requires: MutatingAdmissionPolicy feature enabled in the cluster -apiVersion: admissionregistration.k8s.io/v1alpha1 +apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingAdmissionPolicy metadata: name: inject-daemonset-readiness-tolerations diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0d36c19..51aca32 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,6 +7,7 @@ rules: - apiGroups: - "" resources: + - configmaps - nodes verbs: - create diff --git a/docs/admission-policy.md b/docs/admission-policy.md index 07bcada..414f14a 100644 --- a/docs/admission-policy.md +++ b/docs/admission-policy.md @@ -12,7 +12,7 @@ The MutatingAdmissionPolicy approach uses Kubernetes's native admission control > MutatingAdmissionPolicy is needed to be enabled in the cluster. - Feature gate: `MutatingAdmissionPolicy=true` -- Runtime config: `admissionregistration.k8s.io/v1alpha1=true` +- Runtime config: `admissionregistration.k8s.io/v1beta1=true` - `kubectl` configured to access your cluster - NodeReadinessRule CRDs installed diff --git a/internal/controller/nodereadinessrule_controller.go b/internal/controller/nodereadinessrule_controller.go index aea7417..879be64 100644 --- a/internal/controller/nodereadinessrule_controller.go +++ b/internal/controller/nodereadinessrule_controller.go @@ -90,6 +90,8 @@ func (r *RuleReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) // +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules/status,verbs=get;update;patch // +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;create;update;patch;delete func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) @@ -179,6 +181,7 @@ func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // 1. Deletes the taints associated with the rule. // 2. Remove the rule from the cache. // 3. Remove the finalizer from the rule. +// 4. Sync the Taints ConfigMap func (r *RuleReconciler) reconcileDelete(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) @@ -202,6 +205,13 @@ func (r *RuleReconciler) reconcileDelete(ctx context.Context, rule *readinessv1a if err != nil { return ctrl.Result{}, err } + + // Sync taints to ConfigMap for MutatingAdmissionPolicy + if err := r.Controller.syncTaintsConfigMap(ctx); err != nil { + log.Error(err, "Failed to sync taints configmap", "rule", rule.Name) + // Don't fail reconciliation for this - log and continue + } + return ctrl.Result{}, nil } @@ -690,6 +700,10 @@ func (r *RuleReadinessController) syncTaintsConfigMap(ctx context.Context) error // Extract unique taint keys with readiness.k8s.io/ prefix and NoSchedule effect taintKeysSet := make(map[string]struct{}) for _, rule := range ruleList.Items { + // Skip rules that are being deleted + if !rule.DeletionTimestamp.IsZero() { + continue + } if rule.Spec.Taint.Key != "" && strings.HasPrefix(rule.Spec.Taint.Key, "readiness.k8s.io/") && rule.Spec.Taint.Effect == corev1.TaintEffectNoSchedule { @@ -733,7 +747,7 @@ func (r *RuleReadinessController) syncTaintsConfigMap(ctx context.Context) error } else { // Update existing ConfigMap log.V(1).Info("Updating readiness-taints ConfigMap", "taintCount", len(taintKeys)) - patch := client.MergeFrom(existingCM) + patch := client.MergeFrom(existingCM.DeepCopy()) existingCM.Data = cm.Data if err := r.Patch(ctx, existingCM, patch); err != nil { return fmt.Errorf("failed to update configmap: %w", err) From 95191ab62908e26ab4980b147375b2a42dcb7151 Mon Sep 17 00:00:00 2001 From: pehlicd Date: Sun, 8 Feb 2026 19:02:53 +0100 Subject: [PATCH 3/3] fix: lint issue Signed-off-by: pehlicd --- internal/controller/nodereadinessrule_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/nodereadinessrule_controller.go b/internal/controller/nodereadinessrule_controller.go index 879be64..36e5306 100644 --- a/internal/controller/nodereadinessrule_controller.go +++ b/internal/controller/nodereadinessrule_controller.go @@ -181,7 +181,7 @@ func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // 1. Deletes the taints associated with the rule. // 2. Remove the rule from the cache. // 3. Remove the finalizer from the rule. -// 4. Sync the Taints ConfigMap +// 4. Sync the Taints ConfigMap. func (r *RuleReconciler) reconcileDelete(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx)