Rework control plane policies (#647)

* Switch control plane policies to correct namespace

* Remove high-cpu pool scheduling

* Disable VPA for etcd and kube-apiserver

* Disable HPA for kube-apiserver

* Disable kube-controller-manager client-side rate limiting

* Drop obsolete kyverno tests and CLI

* Update etcd labels

Author: timebertt

Makefile

@@ -82,10 +82,6 @@ test: ## Run unit tests.
 test-integration: $(SETUP_ENVTEST) ## Run integration tests.
 	./hack/test-integration.sh ./test/integration/...
 
-.PHONY: test-kyverno
-test-kyverno: $(KYVERNO) ## Run kyverno policy tests.
-	$(KYVERNO) test --remove-color -v 4 .
-
 .PHONY: test-e2e
 test-e2e: $(GINKGO) ## Run e2e tests.
 	./hack/test-e2e.sh $(GINKGO_FLAGS) ./test/e2e/... ./webhosting-operator/test/e2e/...
@@ -102,7 +98,7 @@ lint: $(GOLANGCI_LINT) ## Run golangci-lint against code.
 	$(GOLANGCI_LINT) run ./... ./webhosting-operator/...
 
 .PHONY: check
-check: lint test test-integration test-kyverno ## Check everything (lint + test + test-integration + test-kyverno).
+check: lint test test-integration ## Check everything (lint + test + test-integration).
 
 .PHONY: verify-fmt
 verify-fmt: fmt ## Verify go code is formatted.

docs/evaluation.md

@@ -124,7 +124,8 @@ In addition to the described components, [kyverno](https://github.com/kyverno/ky
 In the cluster itself, kyverno policies are used for scheduling the sharder and webhosting-operator to the dedicated `sharding` worker pool and experiment to the dedicated `experiment` worker pool.
 This makes sure that these components run on machines isolated from other system components and don't content for compute resources during load tests.
 
-Furthermore, kyverno policies are added to the control plane to ensure a static size of etcd, kube-apiserver, and kube-controller-manager (requests=limits for guaranteed resources, disable vertical autoscaling, 4 replicas of kube-apiserver to disable horizontal autoscaling) and schedule them to a dedicated worker pool using a non-overcommit flavor with more CPU cores per machine.
+Furthermore, kyverno policies are added to the control plane to ensure a static size of etcd, kube-apiserver, and kube-controller-manager (requests=limits for guaranteed resources, disable vertical autoscaling, 4 replicas of kube-apiserver and disable horizontal autoscaling).
+Also, kube-controller-manager's client-side rate limiting is disabled (ref https://github.com/timebertt/kubernetes-controller-sharding/pull/610, [SIG api-machinery recommendation](https://kubernetes.slack.com/archives/C0EG7JC6T/p1680889646346859?thread_ts=1680791299.631439&cid=C0EG7JC6T)) and HTTP/2 is disabled so that API requests are distributed across API server instances (ref https://github.com/gardener/gardener/issues/8810).
 This is done to make load test experiments more stable and their results more reproducible.
 
 ## Measurements

hack/config/policy/controlplane/etcd-main.yaml

@@ -2,7 +2,7 @@ apiVersion: kyverno.io/v1
 kind: Policy
 metadata:
   name: etcd-main
-  namespace: shoot--timebertt--sharding
+  namespace: shoot--ixywdlfvei--sharding
 spec:
   failurePolicy: Fail
   rules:
@@ -15,8 +15,7 @@ spec:
           - Pod
           selector:
             matchLabels:
-              instance: etcd-main
-              name: etcd
+              app.kubernetes.io/name: etcd-main
     mutate:
       patchStrategicMerge:
         spec:
@@ -33,22 +32,16 @@ spec:
             env:
             - name: GOMAXPROCS
               value: "12"
-  # schedule etcd-main on high-cpu worker pool for stable performance
-  - name: add-scheduling-constraints
+  - name: disable-vpa
     match:
       any:
       - resources:
           kinds:
-          - Pod
-          selector:
-            matchLabels:
-              instance: etcd-main
-              name: etcd
+          - VerticalPodAutoscaler
+          names:
+          - etcd-main
     mutate:
-      patchesJson6902: |-
-        - op: add
-          path: "/spec/tolerations/-"
-          value: {"key":"high-cpu","operator":"Equal","value":"true","effect":"NoSchedule"}        
-        - op: replace
-          path: "/spec/affinity/nodeAffinity/requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms"
-          value: [{"matchExpressions": [{"key":"high-cpu","operator":"In","values":["true"]}]}]
+      patchStrategicMerge:
+        spec:
+          updatePolicy:
+            updateMode: Off

hack/config/policy/controlplane/kube-apiserver-scale.yaml

@@ -2,34 +2,24 @@ apiVersion: kyverno.io/v1
 kind: Policy
 metadata:
   name: kube-apiserver
-  namespace: shoot--timebertt--sharding
+  namespace: shoot--ixywdlfvei--sharding
 spec:
   failurePolicy: Fail
   rules:
   # set static replicas on kube-apiserver to ensure similar evaluation environment between load test runs
-  # if the cluster is hibernated (spec.replicas=0), this rule is skipped
-  - name: replicas
+  - name: disable-hpa
     match:
       any:
       - resources:
           kinds:
-          - Deployment
-          selector:
-            matchLabels:
-              app: kubernetes
-              role: apiserver
-    preconditions:
-      all:
-      # Only patch spec.replicas if the control plane is not hibernated, i.e., if spec.replicas>=1.
-      # NB: gardenlet deploys kube-apiserver with spec.replicas=null which is defaulted after the policy webhook call
-      # to spec.replicas=1. Hence, treat spec.replicas=null the same way as spec.replicas=1.
-      - key: "{{ request.object.spec.replicas || `1` }}"
-        operator: GreaterThan
-        value: 0
+          - HorizontalPodAutoscaler
+          names:
+          - kube-apiserver
     mutate:
       patchStrategicMerge:
         spec:
-          replicas: 4
+          minReplicas: 4
+          maxReplicas: 4
   # set static requests/limits on kube-apiserver to ensure similar evaluation environment between load test runs
   - name: resources
     match:
@@ -57,22 +47,16 @@ spec:
             env:
             - name: GOMAXPROCS
               value: "12"
-  # schedule kube-apiserver on high-cpu worker pool for stable performance
-  - name: add-scheduling-constraints
+  - name: disable-vpa
     match:
       any:
       - resources:
           kinds:
-          - Pod
-          selector:
-            matchLabels:
-              app: kubernetes
-              role: apiserver
+          - VerticalPodAutoscaler
+          names:
+          - kube-apiserver-vpa
     mutate:
-      patchesJson6902: |-
-        - op: add
-          path: "/spec/tolerations/-"
-          value: {"key":"high-cpu","operator":"Equal","value":"true","effect":"NoSchedule"}
-        - op: add
-          path: "/spec/affinity/nodeAffinity/requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms"
-          value: [{"matchExpressions": [{"key":"high-cpu","operator":"In","values":["true"]}]}]
+      patchStrategicMerge:
+        spec:
+          updatePolicy:
+            updateMode: Off

hack/config/policy/controlplane/kube-apiserver.yaml

@@ -2,7 +2,7 @@ apiVersion: kyverno.io/v1
 kind: Policy
 metadata:
   name: kube-controller-manager
-  namespace: shoot--timebertt--sharding
+  namespace: shoot--ixywdlfvei--sharding
 spec:
   failurePolicy: Ignore
   rules:
@@ -46,27 +46,8 @@ spec:
         spec:
           updatePolicy:
             updateMode: Off
-  # schedule kube-controller-manager on high-cpu worker pool for stable performance
-  - name: add-scheduling-constraints
-    match:
-      any:
-      - resources:
-          kinds:
-          - Pod
-          selector:
-            matchLabels:
-              app: kubernetes
-              role: controller-manager
-    mutate:
-      patchesJson6902: |-
-        - op: add
-          path: "/spec/tolerations/-"
-          value: {"key":"high-cpu","operator":"Equal","value":"true","effect":"NoSchedule"}        
-        - op: add
-          path: "/spec/affinity/nodeAffinity/requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms"
-          value: [{"matchExpressions": [{"key":"high-cpu","operator":"In","values":["true"]}]}]
-  # increases kube-controller-manager's client-side rate limits to speed up garbage collection after executing load tests
-  - name: increase-rate-limits
+  # disable kube-controller-manager's client-side rate limits similar to webhosting-operator
+  - name: disable-rate-limits
     match:
       any:
       - resources:
@@ -78,10 +59,7 @@ spec:
       patchesJson6902: |-
         - op: add
           path: /spec/template/spec/containers/0/command/-
-          value: "--kube-api-qps=2000"
-        - op: add
-          path: /spec/template/spec/containers/0/command/-
-          value: "--kube-api-burst=2200"
+          value: "--kube-api-qps=-1"
   # disable HTTP2 in kube-controller-manager's so that API requests are distributed across API server instances
   - name: disable-http2
     match:

hack/config/policy/controlplane/kube-controller-manager.yaml

@@ -7,5 +7,4 @@ kind: Kustomization
 resources:
 - etcd-main.yaml
 - kube-apiserver.yaml
-- kube-apiserver-scale.yaml
 - kube-controller-manager.yaml