diff --git a/config/admission-policy/binding.yaml b/config/admission-policy/binding.yaml new file mode 100644 index 0000000..8107aca --- /dev/null +++ b/config/admission-policy/binding.yaml @@ -0,0 +1,15 @@ +--- +# MutatingAdmissionPolicyBinding binds the policy to the ConfigMap parameter +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: inject-daemonset-readiness-tolerations-binding +spec: + policyName: inject-daemonset-readiness-tolerations + # Reference the ConfigMap containing toleration data + paramRef: + name: readiness-taints + namespace: nrr-system + parameterNotFoundAction: Deny + matchResources: + namespaceSelector: {} diff --git a/config/admission-policy/configmap.yaml b/config/admission-policy/configmap.yaml new file mode 100644 index 0000000..eab9101 --- /dev/null +++ b/config/admission-policy/configmap.yaml @@ -0,0 +1,12 @@ +--- +# ConfigMap that stores readiness tolerations +# This will be populated/updated by the NodeReadinessRule controller +apiVersion: v1 +kind: ConfigMap +metadata: + name: readiness-taints + namespace: nrr-system +data: + # Store each toleration key separately for easier CEL access + # Format: key1=readiness.k8s.io/NetworkReady,key2=readiness.k8s.io/StorageReady + taint-keys: "" diff --git a/config/admission-policy/kustomization.yaml b/config/admission-policy/kustomization.yaml new file mode 100644 index 0000000..b3e49bc --- /dev/null +++ b/config/admission-policy/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- configmap.yaml +- policy.yaml +- binding.yaml + +labels: +- pairs: + app.kubernetes.io/name: nrrcontroller + app.kubernetes.io/component: admission-policy diff --git a/config/admission-policy/policy.yaml b/config/admission-policy/policy.yaml new file mode 100644 index 0000000..6da22f1 --- /dev/null +++ b/config/admission-policy/policy.yaml @@ -0,0 +1,71 @@ +--- +# MutatingAdmissionPolicy for automatic DaemonSet toleration injection +# Reads taint keys from a ConfigMap parameter resource +# Requires: MutatingAdmissionPolicy feature enabled in the cluster +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingAdmissionPolicy +metadata: + name: inject-daemonset-readiness-tolerations +spec: + failurePolicy: Fail + + # Define what this policy watches + matchConstraints: + resourceRules: + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["daemonsets"] + + # Reference the ConfigMap that contains toleration data + paramKind: + apiVersion: v1 + kind: ConfigMap + + # Variables for CEL expressions + variables: + # Check if opt-out annotation is set + - name: optedOut + expression: | + has(object.metadata.annotations) && + object.metadata.annotations.exists(k, k == "readiness.k8s.io/auto-tolerate" && object.metadata.annotations[k] == "false") + + # Get existing tolerations (empty array if none) + - name: existingTolerations + expression: | + has(object.spec.template.spec.tolerations) ? + object.spec.template.spec.tolerations : [] + + # Get taint keys from ConfigMap and parse to array + - name: taintKeys + expression: | + ("taint-keys" in params.data) && params.data["taint-keys"] != "" ? + params.data["taint-keys"].split(",") : [] + + # Create tolerations from taint keys (as plain maps since CEL has issues with complex types) + - name: tolerationsToInject + expression: | + variables.taintKeys + .filter(key, !variables.existingTolerations.exists(t, t.key == key)) + .map(key, { + "key": key, + "operator": "Exists", + "effect": "NoSchedule" + }) + + # Apply mutations + mutations: + - patchType: JSONPatch + jsonPatch: + expression: | + !variables.optedOut && size(variables.tolerationsToInject) > 0 ? + [ + JSONPatch{ + op: has(object.spec.template.spec.tolerations) ? "replace" : "add", + path: "/spec/template/spec/tolerations", + value: variables.existingTolerations + variables.tolerationsToInject + } + ] : [] + + # Never reinvoke this policy + reinvocationPolicy: Never diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0d36c19..51aca32 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,6 +7,7 @@ rules: - apiGroups: - "" resources: + - configmaps - nodes verbs: - create diff --git a/docs/admission-policy.md b/docs/admission-policy.md new file mode 100644 index 0000000..414f14a --- /dev/null +++ b/docs/admission-policy.md @@ -0,0 +1,54 @@ +# MutatingAdmissionPolicy for DaemonSet Toleration Injection + +This document describes how to deploy and use the MutatingAdmissionPolicy-based approach for automatically injecting readiness tolerations into DaemonSets. + +## Overview + +The MutatingAdmissionPolicy approach uses Kubernetes's native admission control mechanism with CEL (Common Expression Language) to inject tolerations **without running a webhook server**. This provides a simpler, more declarative alternative to the webhook-based approach. + +## Requirements + +> [!IMPORTANT] +> MutatingAdmissionPolicy is needed to be enabled in the cluster. + +- Feature gate: `MutatingAdmissionPolicy=true` +- Runtime config: `admissionregistration.k8s.io/v1beta1=true` +- `kubectl` configured to access your cluster +- NodeReadinessRule CRDs installed + +## Architecture + +``` +User applies DaemonSet + ↓ +API Server evaluates CEL policy + ↓ +Fetches Tolerations ConfigMap which contains the tolerations to be injected + ↓ +Injects tolerations (if applicable) + ↓ +DaemonSet created with tolerations +``` + +## Deployment + +### Option 1: Using kustomize + +```bash +# Install CRDs first +make install + +# Deploy the admission policy +kubectl apply -k config/admission-policy +``` + +### Option 2: Direct kubectl apply + +```bash +# Install CRDs first +make install + +# Deploy policy and binding +kubectl apply -f config/admission-policy/policy.yaml +kubectl apply -f config/admission-policy/binding.yaml +``` diff --git a/go.mod b/go.mod index 01fca93..9e132dc 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( k8s.io/api v0.34.0 k8s.io/apimachinery v0.34.0 k8s.io/client-go v0.34.0 + k8s.io/klog/v2 v2.130.1 sigs.k8s.io/controller-runtime v0.22.1 ) @@ -91,7 +92,6 @@ require ( k8s.io/apiextensions-apiserver v0.34.0 // indirect k8s.io/apiserver v0.34.0 // indirect k8s.io/component-base v0.34.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect diff --git a/internal/controller/nodereadinessrule_controller.go b/internal/controller/nodereadinessrule_controller.go index 9bb2420..36e5306 100644 --- a/internal/controller/nodereadinessrule_controller.go +++ b/internal/controller/nodereadinessrule_controller.go @@ -90,6 +90,8 @@ func (r *RuleReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) // +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules/status,verbs=get;update;patch // +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;create;update;patch;delete func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) @@ -166,6 +168,12 @@ func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{RequeueAfter: time.Minute}, err } + // Sync taints to ConfigMap for MutatingAdmissionPolicy + if err := r.Controller.syncTaintsConfigMap(ctx); err != nil { + log.Error(err, "Failed to sync taints configmap", "rule", rule.Name) + // Don't fail reconciliation for this - log and continue + } + return ctrl.Result{}, nil } @@ -173,6 +181,7 @@ func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // 1. Deletes the taints associated with the rule. // 2. Remove the rule from the cache. // 3. Remove the finalizer from the rule. +// 4. Sync the Taints ConfigMap. func (r *RuleReconciler) reconcileDelete(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) @@ -196,6 +205,13 @@ func (r *RuleReconciler) reconcileDelete(ctx context.Context, rule *readinessv1a if err != nil { return ctrl.Result{}, err } + + // Sync taints to ConfigMap for MutatingAdmissionPolicy + if err := r.Controller.syncTaintsConfigMap(ctx); err != nil { + log.Error(err, "Failed to sync taints configmap", "rule", rule.Name) + // Don't fail reconciliation for this - log and continue + } + return ctrl.Result{}, nil } @@ -671,6 +687,79 @@ func (r *RuleReadinessController) cleanupNodesAfterSelectorChange(ctx context.Co return nil } +// syncTaintsConfigMap synchronizes readiness taints to a ConfigMap for admission policy. +func (r *RuleReadinessController) syncTaintsConfigMap(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + + // List all NodeReadinessRules + var ruleList readinessv1alpha1.NodeReadinessRuleList + if err := r.List(ctx, &ruleList); err != nil { + return fmt.Errorf("failed to list NodeReadinessRules: %w", err) + } + + // Extract unique taint keys with readiness.k8s.io/ prefix and NoSchedule effect + taintKeysSet := make(map[string]struct{}) + for _, rule := range ruleList.Items { + // Skip rules that are being deleted + if !rule.DeletionTimestamp.IsZero() { + continue + } + if rule.Spec.Taint.Key != "" && + strings.HasPrefix(rule.Spec.Taint.Key, "readiness.k8s.io/") && + rule.Spec.Taint.Effect == corev1.TaintEffectNoSchedule { + taintKeysSet[rule.Spec.Taint.Key] = struct{}{} + } + } + + // Convert set to comma-separated string + taintKeys := make([]string, 0, len(taintKeysSet)) + for key := range taintKeysSet { + taintKeys = append(taintKeys, key) + } + taintKeysStr := strings.Join(taintKeys, ",") + + // Update or create ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "readiness-taints", + Namespace: "nrr-system", + }, + } + + // Try to get existing ConfigMap + existingCM := &corev1.ConfigMap{} + err := r.Get(ctx, client.ObjectKey{Name: "readiness-taints", Namespace: "nrr-system"}, existingCM) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get configmap: %w", err) + } + + // Set data + cm.Data = map[string]string{ + "taint-keys": taintKeysStr, + } + + if apierrors.IsNotFound(err) { + // Create new ConfigMap + log.Info("Creating readiness-taints ConfigMap", "taintCount", len(taintKeys)) + if err := r.Create(ctx, cm); err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + } else { + // Update existing ConfigMap + log.V(1).Info("Updating readiness-taints ConfigMap", "taintCount", len(taintKeys)) + patch := client.MergeFrom(existingCM.DeepCopy()) + existingCM.Data = cm.Data + if err := r.Patch(ctx, existingCM, patch); err != nil { + return fmt.Errorf("failed to update configmap: %w", err) + } + } + + log.V(2).Info("Successfully synced taints to ConfigMap", + "totalRules", len(ruleList.Items), + "readinessTaints", len(taintKeys)) + return nil +} + func (r *RuleReconciler) ensureFinalizer(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule, finalizer string) (finalizerAdded bool, err error) { // Finalizers can only be added when the deletionTimestamp is not set. if !rule.GetDeletionTimestamp().IsZero() {