test: using MutatingAdmissionPolicy to schedule CS pods (#1777)

This change documents a proof-of-concept of configuring the node
placement of Config Sync system pods using MutatingAdmissionPolicy.

This works out of the box, but is currently an alpha api and is behind a
feature gate. It is targeted for beta in k8s 1.34.

Author: sdowell

e2e/nomostest/clusters/kind.go

@@ -26,6 +26,7 @@ import (
 	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/docker"
 	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/taskgroup"
 	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/testing"
+	"k8s.io/apiserver/pkg/features"
 	"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	"sigs.k8s.io/kind/pkg/cluster"
 )
@@ -155,6 +156,15 @@ func createKindCluster(p *cluster.Provider, name, kcfgPath string) error {
 					fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:%d"]
   endpoint = ["http://%s:%d"]`, docker.RegistryPort, docker.RegistryName, docker.RegistryPort),
 				},
+				// MutatingAdmissionPolicy is currently off by default at the time of writing.
+				// It is enabled so that it can be tested with Config Sync.
+				// The feature gate can be removed once MutatingAdmissionPolicy enabled by default.
+				FeatureGates: map[string]bool{
+					string(features.MutatingAdmissionPolicy): true,
+				},
+				RuntimeConfig: map[string]string{
+					"admissionregistration.k8s.io/v1beta1": "true",
+				},
 				// Enable ValidatingAdmissionWebhooks in the Kind cluster, as these
 				// are disabled by default.
 				// Also mount etcd to tmpfs for memory-backed storage.
@@ -166,7 +176,7 @@ etcd:
     dataDir: /tmp/etcd
 apiServer:
   extraArgs:
-    "enable-admission-plugins": "ValidatingAdmissionWebhook"`,
+    "enable-admission-plugins": "ValidatingAdmissionWebhook,MutatingAdmissionPolicy,ValidatingAdmissionPolicy"`,
 				},
 			}),
 			// Retain nodes for debugging logs.

e2e/nomostest/config_sync.go

@@ -1410,6 +1410,23 @@ func podHasReadyCondition(conditions []corev1.PodCondition) bool {
 	return false
 }
 
+// ValidatePodByLabel validates that all Pods matching the provided label pass the
+// provided list of predicates.
+func ValidatePodByLabel(nt *NT, label, value string, predicates ...testpredicates.Predicate) error {
+	newPods := &corev1.PodList{}
+	if err := nt.KubeClient.List(newPods, client.InNamespace(configmanagement.ControllerNamespace), client.MatchingLabels{label: value}); err != nil {
+		return err
+	}
+	tg := taskgroup.New()
+	for _, pod := range newPods.Items {
+		po := pod
+		tg.Go(func() error {
+			return nt.Validate(po.Name, po.Namespace, &corev1.Pod{}, predicates...)
+		})
+	}
+	return tg.Wait()
+}
+
 // NewPodReady checks if the new created pods are ready.
 // It also checks if the new children pods that are managed by the pods are ready.
 func NewPodReady(nt *NT, labelName, currentLabel, childLabel string, oldCurrentPods, oldChildPods []corev1.Pod) error {

e2e/nomostest/testpredicates/predicates.go

@@ -224,6 +224,28 @@ func HasExactlyLabelKeys(wantKeys ...string) Predicate {
 	}
 }
 
+// HasExactlyNodeAffinity ensures the Pod has the provided nodeAffinity
+func HasExactlyNodeAffinity(nodeAffinity *corev1.NodeAffinity) Predicate {
+	return func(o client.Object) error {
+		if o == nil {
+			return ErrObjectNotFound
+		}
+		pod, ok := o.(*corev1.Pod)
+		if !ok {
+			return WrongTypeErr(pod, &corev1.Pod{})
+		}
+		gotAffinity := &corev1.NodeAffinity{}
+		if pod.Spec.Affinity != nil && pod.Spec.Affinity.NodeAffinity != nil {
+			gotAffinity = pod.Spec.Affinity.NodeAffinity
+		}
+		if !equality.Semantic.DeepEqual(nodeAffinity, gotAffinity) {
+			return fmt.Errorf("expected %s to have spec.affinity.nodeAffinity: %s, but got %s",
+				kinds.ObjectSummary(pod), log.AsJSON(nodeAffinity), log.AsJSON(gotAffinity))
+		}
+		return nil
+	}
+}
+
 // HasExactlyImage ensures a container has the expected image.
 func HasExactlyImage(containerName, expectImageName, expectImageTag, expectImageDigest string) Predicate {
 	return func(o client.Object) error {

e2e/testcases/mutating_admission_policy_test.go

@@ -0,0 +1,100 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/GoogleContainerTools/config-sync/e2e/nomostest"
+	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/ntopts"
+	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/syncsource"
+	nomostesting "github.com/GoogleContainerTools/config-sync/e2e/nomostest/testing"
+	"github.com/GoogleContainerTools/config-sync/e2e/nomostest/testpredicates"
+	"github.com/GoogleContainerTools/config-sync/pkg/api/configsync"
+	"github.com/GoogleContainerTools/config-sync/pkg/core/k8sobjects"
+	"github.com/GoogleContainerTools/config-sync/pkg/kinds"
+	"github.com/GoogleContainerTools/config-sync/pkg/reconcilermanager"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// This test currently requires KinD because MutatingAdmissionPolicy is alpha
+// and requires a feature gate.
+func TestMutatingAdmissionPolicy(t *testing.T) {
+	nt := nomostest.New(t, nomostesting.Reconciliation2,
+		ntopts.SyncWithGitSource(nomostest.DefaultRootSyncID, ntopts.Unstructured),
+		ntopts.RequireKind(t))
+	rootSyncGitRepo := nt.SyncSourceGitReadWriteRepository(nomostest.DefaultRootSyncID)
+
+	mapFile := filepath.Join(".", "..", "..", "examples", "mutating-admission-policies", "config-sync-node-placement.yaml")
+	nt.T.Cleanup(func() {
+		nt.Must(nt.Shell.Kubectl("delete", "--ignore-not-found", "-f", mapFile))
+		nt.Must(nomostest.Wait(nt.T, "reconciler-manager has no nodeAffinity", time.Minute, func() error {
+			if err := nomostest.DeletePodByLabel(nt, "app", reconcilermanager.ManagerName, false); err != nil {
+				return err
+			}
+			return nomostest.ValidatePodByLabel(nt, "app", reconcilermanager.ManagerName,
+				testpredicates.HasExactlyNodeAffinity(&corev1.NodeAffinity{}))
+		}))
+	})
+	nt.Must(nt.Shell.Kubectl("apply", "-f", mapFile))
+
+	// expected nodeAffinity from the example MutatingAdmissionPolicy yaml
+	exampleNodeAffinity := &corev1.NodeAffinity{
+		PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
+			{
+				Weight: 1,
+				Preference: corev1.NodeSelectorTerm{
+					MatchExpressions: []corev1.NodeSelectorRequirement{
+						{
+							Key:      "another-node-label-key",
+							Operator: corev1.NodeSelectorOpIn,
+							Values:   []string{"another-node-label-value"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// bounce reconciler-manager Pod and verify the nodeAffinity is applied by MAP
+	nt.Must(nomostest.ValidatePodByLabel(nt, "app", reconcilermanager.ManagerName,
+		testpredicates.HasExactlyNodeAffinity(&corev1.NodeAffinity{})))
+	nt.T.Log("Replacing the reconciler-manager Pod to validate nodeAffinity is added")
+	// MutatingAdmissionPolicy does not surface a readiness status, so need to retry.
+	nt.Must(nomostest.Wait(nt.T, "reconciler-manager has mutated nodeAffinity", time.Minute, func() error {
+		if err := nomostest.DeletePodByLabel(nt, "app", reconcilermanager.ManagerName, false); err != nil {
+			return err
+		}
+		return nomostest.ValidatePodByLabel(nt, "app", reconcilermanager.ManagerName,
+			testpredicates.HasExactlyNodeAffinity(exampleNodeAffinity))
+	}))
+
+	// update the RootSync to trigger a Deployment change, verify new reconciler Pod has nodeAffinity
+	rootSync := nomostest.RootSyncObjectV1Beta1FromRootRepo(nt, nomostest.DefaultRootSyncID.Name)
+	rootSync.Spec.Git.Dir = "foo"
+	nt.Must(nt.KubeClient.Apply(rootSync))
+	nt.Must(rootSyncGitRepo.Add("foo/ns.yaml", k8sobjects.NamespaceObject("test-map-ns")))
+	nt.Must(rootSyncGitRepo.CommitAndPush("add foo-ns under foo/ dir"))
+	nt.Must(nt.WatchForSync(kinds.RootSyncV1Beta1(), rootSync.Name, configsync.ControllerNamespace,
+		&syncsource.GitSyncSource{
+			ExpectedCommit:    rootSyncGitRepo.MustHash(t),
+			ExpectedDirectory: "foo",
+		}))
+	nt.Must(nt.Validate("test-map-ns", "", &corev1.Namespace{}))
+	nt.Must(nomostest.ValidatePodByLabel(nt, "app", reconcilermanager.Reconciler,
+		testpredicates.HasExactlyNodeAffinity(exampleNodeAffinity)))
+}

examples/mutating-admission-policies/README.md

@@ -0,0 +1,21 @@
+# Scheduling Config Sync system pods using MutatingAdmissionPolicies
+
+[MutatingAdmissionPolicy] can be used to configure the way the Config Sync's
+system Pods are scheduled.
+
+The [example in this directory](./config-sync-node-placement.yaml) demonstrates
+how to inject `nodeAffinity` into all Pods in the `config-management-system` Namespace.
+
+Note that this is just a demonstrative example, and different `matchConstraints` and
+`mutations` can be applied depending on the use case.
+
+Caveats:
+
+- MutatingAdmissionPolicy is currently in alpha and requires feature gate enablement. It is [targeted to enter beta in k8s 1.34].
+- MutatingAdmissionPolicy does not update existing Pods. Pods created before the policy was applied must be updated/recreated.
+
+
+
+
+[MutatingAdmissionPolicy]: https://kubernetes.io/docs/reference/access-authn-authz/mutating-admission-policy/
+[targeted to enter beta in k8s 1.34]: https://github.com/kubernetes/enhancements/issues/3962

examples/mutating-admission-policies/config-sync-node-placement.yaml

@@ -0,0 +1,70 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingAdmissionPolicy
+metadata:
+  name: "configsync-nodeplacement"
+spec:
+  # This example uniformly applies the nodeAffinity to *all* Pods in config-management-system.
+  # Different matchConstraints can be used for more granular mutation.
+  matchConstraints:
+    namespaceSelector:
+      matchLabels:
+        kubernetes.io/metadata.name: config-management-system
+    resourceRules:
+    - apiGroups:   [""]
+      apiVersions: ["v1"]
+      operations:  ["CREATE", "UPDATE"]
+      resources:   ["pods"]
+  failurePolicy: Fail
+  reinvocationPolicy: IfNeeded
+  mutations:
+  # Simple example of adding nodeAffinity: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
+  # Similar mutations can be applied for nodeSelector/tolerations/etc
+  - patchType: "JSONPatch"
+    jsonPatch:
+      expression: >
+        [
+          JSONPatch{
+            op: "add", path: "/spec/affinity",
+            value: Object.spec.affinity{
+              nodeAffinity: Object.spec.affinity.nodeAffinity{
+                preferredDuringSchedulingIgnoredDuringExecution: [
+                  Object.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution{
+                    weight: 1,
+                    preference: Object.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.preference{
+                      matchExpressions: [
+                        Object.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.preference.matchExpressions{
+                          key: "another-node-label-key",
+                          operator: "In",
+                          values: [
+                            "another-node-label-value"
+                          ]
+                        }
+                      ]
+                    }
+                  }
+                ]
+              }
+            }
+          }
+        ]
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingAdmissionPolicyBinding
+metadata:
+  name: "configsync-nodeplacement"
+spec:
+  policyName: "configsync-nodeplacement"

go.mod

@@ -43,6 +43,7 @@ require (
 	k8s.io/api v0.34.1
 	k8s.io/apiextensions-apiserver v0.34.1
 	k8s.io/apimachinery v0.34.1
+	k8s.io/apiserver v0.34.1
 	k8s.io/cli-runtime v0.34.1
 	k8s.io/client-go v0.34.1
 	k8s.io/cluster-registry v0.0.6
@@ -173,7 +174,6 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiserver v0.34.1 // indirect
 	k8s.io/component-base v0.34.1 // indirect
 	k8s.io/component-helpers v0.34.1 // indirect
 	k8s.io/controller-manager v0.34.0 // indirect