diff --git a/vertical-pod-autoscaler/deploy/vpa-v1-crd-gen.yaml b/vertical-pod-autoscaler/deploy/vpa-v1-crd-gen.yaml index 70adb552bb39..3f98413f7fb4 100644 --- a/vertical-pod-autoscaler/deploy/vpa-v1-crd-gen.yaml +++ b/vertical-pod-autoscaler/deploy/vpa-v1-crd-gen.yaml @@ -372,6 +372,22 @@ spec: - Auto - "Off" type: string + oomBumpUpRatio: + anyOf: + - type: integer + - type: string + description: oomBumpUpRatio is the ratio to increase memory + when OOM is detected. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + oomMinBumpUp: + anyOf: + - type: integer + - type: string + description: oomMinBumpUp is the minimum increase in memory + when OOM is detected. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object type: array type: object diff --git a/vertical-pod-autoscaler/docs/api.md b/vertical-pod-autoscaler/docs/api.md index f7e03b0611c0..eb981c0ee347 100644 --- a/vertical-pod-autoscaler/docs/api.md +++ b/vertical-pod-autoscaler/docs/api.md @@ -48,6 +48,8 @@ _Appears in:_ | `maxAllowed` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcelist-v1-core)_ | Specifies the maximum amount of resources that will be recommended
for the container. The default is no maximum. | | | | `controlledResources` _[ResourceName](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcename-v1-core)_ | Specifies the type of recommendations that will be computed
(and possibly applied) by VPA.
If not specified, the default of [ResourceCPU, ResourceMemory] will be used. | | | | `controlledValues` _[ContainerControlledValues](#containercontrolledvalues)_ | Specifies which resource values should be controlled.
The default is "RequestsAndLimits". | | Enum: [RequestsAndLimits RequestsOnly]
| +| `oomBumpUpRatio` _float_ | OOMBumpUpRatio is the ratio to increase resources when OOM is detected. | | Minimum: 1
| +| `oomMinBumpUp` _float_ | OOMMinBumpUp is the minimum increase in resources when OOM is detected. | | Minimum: 0
| #### ContainerScalingMode diff --git a/vertical-pod-autoscaler/docs/flags.md b/vertical-pod-autoscaler/docs/flags.md index feb3dc32e371..ba8826af59cb 100644 --- a/vertical-pod-autoscaler/docs/flags.md +++ b/vertical-pod-autoscaler/docs/flags.md @@ -14,7 +14,7 @@ This document is auto-generated from the flag definitions in the VPA admission-c | `address` | string | ":8944" | The address to expose Prometheus metrics. | | `alsologtostderr` | | | log to standard error as well as files (no effect when -logtostderr=true) | | `client-ca-file` | string | "/etc/tls-certs/caCert.pem" | Path to CA PEM file. | -| `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
InPlaceOrRecreate=true\|false (BETA - default=true) | +| `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
InPlaceOrRecreate=true\|false (BETA - default=true)
PerVPAConfig=true\|false (ALPHA - default=false) | | `ignored-vpa-object-namespaces` | string | | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. | | `kube-api-burst` | float | 100 | QPS burst limit when making requests to Kubernetes apiserver | | `kube-api-qps` | float | 50 | QPS limit when making requests to Kubernetes apiserver | @@ -68,7 +68,7 @@ This document is auto-generated from the flag definitions in the VPA recommender | `cpu-integer-post-processor-enabled` | | | Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental) | | `external-metrics-cpu-metric` | string | | ALPHA. Metric to use with external metrics provider for CPU usage. | | `external-metrics-memory-metric` | string | | ALPHA. Metric to use with external metrics provider for memory usage. | -| `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
InPlaceOrRecreate=true\|false (BETA - default=true) | +| `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
InPlaceOrRecreate=true\|false (BETA - default=true)
PerVPAConfig=true\|false (ALPHA - default=false) | | `history-length` | string | "8d" | How much time back prometheus have to be queried to get historical metrics | | `history-resolution` | string | "1h" | Resolution at which Prometheus is queried for historical metrics | | `humanize-memory` | | | DEPRECATED: Convert memory values in recommendations to the highest appropriate SI unit with up to 2 decimal places for better readability. This flag is deprecated and will be removed in a future version. Use --round-memory-bytes instead. | @@ -95,8 +95,8 @@ This document is auto-generated from the flag definitions in the VPA recommender | `metric-for-pod-labels` | string | "up{job=\"kubernetes-pods\"}" | Which metric to look for pod labels in metrics | | `min-checkpoints` | int | 10 | Minimum number of checkpoints to write per recommender's main loop. WARNING: this flag is deprecated and doesn't have any effect. It will be removed in a future release. Refer to update-worker-count to influence the minimum number of checkpoints written per loop. | | `one-output` | severity | | If true, only write logs to their native level (vs also writing to each lower severity level; no effect when -logtostderr=true) | -| `oom-bump-up-ratio` | float | 1.2 | The memory bump up ratio when OOM occurred, default is 1.2. | -| `oom-min-bump-up-bytes` | float | 1.048576e+08 | The minimal increase of memory when OOM occurred in bytes, default is 100 * 1024 * 1024 | +| `oom-bump-up-ratio` | float | 1.2 | Default memory bump up ratio when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 1.2. | +| `oom-min-bump-up-bytes` | float | 1.048576e+08 | Default minimal increase of memory (in bytes) when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 100 * 1024 * 1024 (100Mi). | | `password` | string | | The password used in the prometheus server basic auth | | `pod-label-prefix` | string | "pod_label_" | Which prefix to look for pod labels in metrics | | `pod-name-label` | string | "kubernetes_pod_name" | Label name to look for pod names | @@ -144,7 +144,7 @@ This document is auto-generated from the flag definitions in the VPA updater cod | `eviction-rate-burst` | int | 1 | Burst of pods that can be evicted. | | `eviction-rate-limit` | float | | Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable
the rate limiter. (default -1) | | `eviction-tolerance` | float | 0.5 | Fraction of replica count that can be evicted for update, if more than one pod can be evicted. | -| `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
InPlaceOrRecreate=true\|false (BETA - default=true) | +| `feature-gates` | mapStringBool | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
AllAlpha=true\|false (ALPHA - default=false)
AllBeta=true\|false (BETA - default=false)
InPlaceOrRecreate=true\|false (BETA - default=true)
PerVPAConfig=true\|false (ALPHA - default=false) | | `ignored-vpa-object-namespaces` | string | | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. | | `in-recommendation-bounds-eviction-lifetime-threshold` | | 12h0m0s | duration Pods that live for at least that long can be evicted even if their request is within the [MinRecommended...MaxRecommended] range | | `kube-api-burst` | float | 100 | QPS burst limit when making requests to Kubernetes apiserver | diff --git a/vertical-pod-autoscaler/e2e/v1/admission_controller.go b/vertical-pod-autoscaler/e2e/v1/admission_controller.go index 9323b5a96d51..0812de856d79 100644 --- a/vertical-pod-autoscaler/e2e/v1/admission_controller.go +++ b/vertical-pod-autoscaler/e2e/v1/admission_controller.go @@ -882,26 +882,149 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { err := InstallRawVPA(f, validVPA) gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Valid VPA object rejected") - ginkgo.By("Setting up invalid VPA object") - // The invalid object differs by name and minAllowed - there is an invalid "requests" field. - invalidVPA := []byte(`{ - "kind": "VerticalPodAutoscaler", - "apiVersion": "autoscaling.k8s.io/v1", - "metadata": {"name": "hamster-vpa-invalid"}, - "spec": { - "targetRef": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name":"hamster" - }, - "resourcePolicy": { - "containerPolicies": [{"containerName": "*", "minAllowed":{"requests":{"cpu":"50m"}}}] - } - } - }`) - err2 := InstallRawVPA(f, invalidVPA) - gomega.Expect(err2).To(gomega.HaveOccurred(), "Invalid VPA object accepted") - gomega.Expect(err2.Error()).To(gomega.MatchRegexp(`.*admission webhook .*vpa.* denied the request: .*`)) + ginkgo.By("Setting up invalid VPA objects") + testCases := []struct { + name string + vpaJSON string + expectedErr string + }{ + { + name: "Invalid oomBumpUpRatio (negative value)", + vpaJSON: `{ + "apiVersion": "autoscaling.k8s.io/v1", + "kind": "VerticalPodAutoscaler", + "metadata": {"name": "oom-test-vpa"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "oom-test" + }, + "updatePolicy": { + "updateMode": "Auto" + }, + "resourcePolicy": { + "containerPolicies": [{ + "containerName": "*", + "oomBumpUpRatio": -1, + "oomMinBumpUp": 104857600 + }] + } + } + }`, + expectedErr: "spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio: Invalid value: -1: spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio in body should be greater than or equal to 1", + }, + { + name: "Invalid oomBumpUpRatio (string value)", + vpaJSON: `{ + "apiVersion": "autoscaling.k8s.io/v1", + "kind": "VerticalPodAutoscaler", + "metadata": {"name": "oom-test-vpa"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "oom-test" + }, + "updatePolicy": { + "updateMode": "Auto" + }, + "resourcePolicy": { + "containerPolicies": [{ + "containerName": "*", + "oomBumpUpRatio": "12", + "oomMinBumpUp": 104857600 + }] + } + } + }`, + expectedErr: "json: cannot unmarshal string into Go struct field ContainerResourcePolicy.spec.resourcePolicy.containerPolicies.oomBumpUpRatio of type float64", + }, + { + name: "Invalid oomBumpUpRatio (less than 1)", + vpaJSON: `{ + "apiVersion": "autoscaling.k8s.io/v1", + "kind": "VerticalPodAutoscaler", + "metadata": {"name": "oom-test-vpa"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "oom-test" + }, + "updatePolicy": { + "updateMode": "Auto" + }, + "resourcePolicy": { + "containerPolicies": [{ + "containerName": "*", + "oomBumpUpRatio": 0.5, + "oomMinBumpUp": 104857600 + }] + } + } + }`, + expectedErr: "spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio: Invalid value: 0.5: spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio in body should be greater than or equal to 1", + }, + { + name: "Invalid oomMinBumpUp (negative value)", + vpaJSON: `{ + "apiVersion": "autoscaling.k8s.io/v1", + "kind": "VerticalPodAutoscaler", + "metadata": {"name": "oom-test-vpa"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "oom-test" + }, + "updatePolicy": { + "updateMode": "Auto" + }, + "resourcePolicy": { + "containerPolicies": [{ + "containerName": "*", + "oomBumpUpRatio": 2, + "oomMinBumpUp": -1 + }] + } + } + }`, + expectedErr: "spec.resourcePolicy.containerPolicies[0].oomMinBumpUp: Invalid value: -1: spec.resourcePolicy.containerPolicies[0].oomMinBumpUp in body should be greater than or equal to 0", + }, + { + name: "Invalid minAllowed (invalid requests field)", + vpaJSON: `{ + "apiVersion": "autoscaling.k8s.io/v1", + "kind": "VerticalPodAutoscaler", + "metadata": {"name": "hamster-vpa-invalid"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "hamster" + }, + "resourcePolicy": { + "containerPolicies": [{ + "containerName": "*", + "minAllowed": { + "requests": { + "cpu": "50m" + } + } + }] + } + } + }`, + expectedErr: "admission webhook .*vpa.* denied the request:", + }, + } + for _, tc := range testCases { + ginkgo.By(fmt.Sprintf("Testing %s", tc.name)) + err := InstallRawVPA(f, []byte(tc.vpaJSON)) + gomega.Expect(err).To(gomega.HaveOccurred(), "Invalid VPA object accepted") + gomega.Expect(err.Error()).To(gomega.MatchRegexp(tc.expectedErr)) + } }) ginkgo.It("reloads the webhook leaf and CA certificate", func(ctx ginkgo.SpecContext) { diff --git a/vertical-pod-autoscaler/e2e/v1/common.go b/vertical-pod-autoscaler/e2e/v1/common.go index 7d54a0ffa4e7..225442bfa9d3 100644 --- a/vertical-pod-autoscaler/e2e/v1/common.go +++ b/vertical-pod-autoscaler/e2e/v1/common.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" @@ -612,6 +613,20 @@ func WaitForPodsUpdatedWithoutEviction(f *framework.Framework, initialPods *apiv return err } +// checkPerVPAConfigTestsEnabled checks if the PerVPAConfig feature gate is enabled +// in the VPA recommender. +func checkPerVPAConfigTestsEnabled(f *framework.Framework) { + ginkgo.By("Checking PerVPAConfig feature gate is enabled for recommender") + deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-recommender", metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1)) + vpaRecommenderPod := deploy.Spec.Template.Spec.Containers[0] + gomega.Expect(vpaRecommenderPod.Name).To(gomega.Equal("recommender")) + if !anyContainsSubstring(vpaRecommenderPod.Args, fmt.Sprintf("%s=true", string(features.PerVPAConfig))) { + ginkgo.Skip("Skipping suite: PerVPAConfig feature gate is not enabled for the VPA recommender") + } +} + func anyContainsSubstring(arr []string, substr string) bool { for _, s := range arr { if strings.Contains(s, substr) { diff --git a/vertical-pod-autoscaler/e2e/v1/recommender.go b/vertical-pod-autoscaler/e2e/v1/recommender.go index 694f257b3cc4..f4bc2c0aaf90 100644 --- a/vertical-pod-autoscaler/e2e/v1/recommender.go +++ b/vertical-pod-autoscaler/e2e/v1/recommender.go @@ -24,6 +24,7 @@ import ( autoscaling "k8s.io/api/autoscaling/v1" apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" @@ -411,6 +412,64 @@ var _ = RecommenderE2eDescribe("VPA CRD object", func() { }) }) +var _ = RecommenderE2eDescribe("OOM with custom config", ginkgo.Label("FG:PerVPAConfig"), func() { + const replicas = 3 + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + var ( + vpaCRD *vpa_types.VerticalPodAutoscaler + vpaClientSet vpa_clientset.Interface + ) + ginkgo.BeforeEach(func() { + checkPerVPAConfigTestsEnabled(f) + ns := f.Namespace.Name + vpaClientSet = getVpaClientSet(f) + ginkgo.By("Setting up a hamster deployment") + runOomingReplicationController( + f.ClientSet, + ns, + "hamster", + replicas) + ginkgo.By("Setting up a VPA CRD") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "v1", + Kind: "Deployment", + Name: "hamster", + } + containerName := GetHamsterContainerNameByIndex(0) + vpaCRD = test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(targetRef). + WithContainer(containerName). + WithOOMBumpUpRatio(resource.NewQuantity(2, resource.DecimalSI)). + Get() + InstallVPA(f, vpaCRD) + }) + ginkgo.It("have memory requests growing with OOMs more than the default", func() { + listOptions := metav1.ListOptions{ + LabelSelector: "name=hamster", + FieldSelector: getPodSelectorExcludingDonePodsOrDie(), + } + err := waitForResourceRequestInRangeInPods( + f, oomTestTimeout, listOptions, apiv1.ResourceMemory, + ParseQuantityOrDie("1024Mi"), ParseQuantityOrDie("1024Mi")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Waiting for recommendation to be filled") + vpa, err := WaitForRecommendationPresent(vpaClientSet, vpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1)) + + currentMemory := vpa.Status.Recommendation.ContainerRecommendations[0].Target.Memory().Value() + oomReplicationControllerRequestLimit := int64(1024 * 1024 * 1024) // from runOomingReplicationController + defaultBumpMemory := float64(oomReplicationControllerRequestLimit) * 1.2 // DefaultOOMBumpUpRatio + customBumpMemory := float64(oomReplicationControllerRequestLimit) * 2.0 // Custom ratio from VPA config + + gomega.Expect(currentMemory).Should(gomega.BeNumerically(">", int64(defaultBumpMemory)), + fmt.Sprintf("Memory recommendation should be at bigger than default bump up ratio (2x). Got: %d, Expected: >= %d", currentMemory, int64(customBumpMemory))) + }) +}) + func deleteRecommender(c clientset.Interface) error { namespace := "kube-system" listOptions := metav1.ListOptions{} diff --git a/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler.go b/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler.go index f7bed6397d94..3acb9515a764 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler.go +++ b/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler.go @@ -136,6 +136,28 @@ func ValidateVPA(vpa *vpa_types.VerticalPodAutoscaler, isCreate bool) error { if policy.ContainerName == "" { return fmt.Errorf("containerPolicies.ContainerName is required") } + + // check that perVPA is on if being used + if err := validatePerVPAFeatureFlag(&policy); err != nil { + return err + } + + // Validate OOMBumpUpRatio + if policy.OOMBumpUpRatio != nil { + ratio := float64(policy.OOMBumpUpRatio.MilliValue()) / 1000.0 + if ratio < 1.0 { + return fmt.Errorf("OOMBumpUpRatio must be greater than or equal to 1.0, got %v", ratio) + } + } + + // Validate OOMMinBumpUp + if policy.OOMMinBumpUp != nil { + minBump := policy.OOMMinBumpUp.Value() + if minBump < 0 { + return fmt.Errorf("OOMMinBumpUp must be greater than or equal to 0, got %v bytes", minBump) + } + } + mode := policy.Mode if mode != nil { if _, found := possibleScalingModes[*mode]; !found { @@ -200,3 +222,12 @@ func validateMemoryResolution(val apires.Quantity) error { } return nil } + +func validatePerVPAFeatureFlag(policy *vpa_types.ContainerResourcePolicy) error { + featureFlagOn := features.Enabled(features.PerVPAConfig) + perVPA := policy.OOMBumpUpRatio != nil || policy.OOMMinBumpUp != nil + if !featureFlagOn && perVPA { + return fmt.Errorf("OOMBumpUpRatio and OOMMinBumpUp are not supported when feature flag %s is disabled", features.PerVPAConfig) + } + return nil +} diff --git a/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler_test.go b/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler_test.go index 9074f37dffbd..78044ff61b9f 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler_test.go +++ b/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler_test.go @@ -51,6 +51,7 @@ func TestValidateVPA(t *testing.T) { isCreate bool expectError error inPlaceOrRecreateFeatureGateDisabled bool + PerVPAConfigDisabled bool }{ { name: "empty update", @@ -319,10 +320,64 @@ func TestValidateVPA(t *testing.T) { }, }, }, + { + name: "per-vpa config active and used", + vpa: vpa_types.VerticalPodAutoscaler{ + Spec: vpa_types.VerticalPodAutoscalerSpec{ + UpdatePolicy: &vpa_types.PodUpdatePolicy{ + UpdateMode: &validUpdateMode, + }, + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + Mode: &validScalingMode, + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("10"), + }, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("100"), + }, + OOMBumpUpRatio: resource.NewQuantity(2, resource.DecimalSI), + }, + }, + }, + }, + }, + PerVPAConfigDisabled: false, + }, + { + name: "per-vpa config disabled and used", + vpa: vpa_types.VerticalPodAutoscaler{ + Spec: vpa_types.VerticalPodAutoscalerSpec{ + UpdatePolicy: &vpa_types.PodUpdatePolicy{ + UpdateMode: &validUpdateMode, + }, + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + Mode: &validScalingMode, + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("10"), + }, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("100"), + }, + OOMMinBumpUp: resource.NewQuantity(2, resource.DecimalSI), + }, + }, + }, + }, + }, + PerVPAConfigDisabled: true, + expectError: fmt.Errorf("OOMBumpUpRatio and OOMMinBumpUp are not supported when feature flag PerVPAConfig is disabled"), + }, } for _, tc := range tests { t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, features.MutableFeatureGate, features.InPlaceOrRecreate, !tc.inPlaceOrRecreateFeatureGateDisabled) + featuregatetesting.SetFeatureGateDuringTest(t, features.MutableFeatureGate, features.PerVPAConfig, !tc.PerVPAConfigDisabled) err := ValidateVPA(&tc.vpa, tc.isCreate) if tc.expectError == nil { assert.NoError(t, err) diff --git a/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go b/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go index 6ae164ce4cac..7d6414563db1 100644 --- a/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go +++ b/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go @@ -19,6 +19,7 @@ package v1 import ( autoscaling "k8s.io/api/autoscaling/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -221,6 +222,14 @@ type ContainerResourcePolicy struct { // The default is "RequestsAndLimits". // +optional ControlledValues *ContainerControlledValues `json:"controlledValues,omitempty" protobuf:"bytes,6,rep,name=controlledValues"` + + // oomBumpUpRatio is the ratio to increase memory when OOM is detected. + // +optional + OOMBumpUpRatio *resource.Quantity `json:"oomBumpUpRatio,omitempty" protobuf:"bytes,7,opt,name=oomBumpUpRatio"` + + // oomMinBumpUp is the minimum increase in memory when OOM is detected. + // +optional + OOMMinBumpUp *resource.Quantity `json:"oomMinBumpUp,omitempty" protobuf:"bytes,8,opt,name=oomMinBumpUp"` } const ( diff --git a/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/zz_generated.deepcopy.go b/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/zz_generated.deepcopy.go index 732ebc2377ba..86ff44f69ea1 100644 --- a/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/zz_generated.deepcopy.go +++ b/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/zz_generated.deepcopy.go @@ -63,6 +63,16 @@ func (in *ContainerResourcePolicy) DeepCopyInto(out *ContainerResourcePolicy) { *out = new(ContainerControlledValues) **out = **in } + if in.OOMBumpUpRatio != nil { + in, out := &in.OOMBumpUpRatio, &out.OOMBumpUpRatio + x := (*in).DeepCopy() + *out = &x + } + if in.OOMMinBumpUp != nil { + in, out := &in.OOMMinBumpUp, &out.OOMMinBumpUp + x := (*in).DeepCopy() + *out = &x + } return } diff --git a/vertical-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go b/vertical-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go index a8b101ba6bb5..a844d751367f 100644 --- a/vertical-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/vertical-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -19,6 +19,7 @@ limitations under the License. package fake import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" @@ -55,9 +56,13 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset { cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} cs.AddReactor("*", "*", testing.ObjectReaction(o)) cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchActcion, ok := action.(testing.WatchActionImpl); ok { + opts = watchActcion.ListOptions + } gvr := action.GetResource() ns := action.GetNamespace() - watch, err := o.Watch(gvr, ns) + watch, err := o.Watch(gvr, ns, opts) if err != nil { return false, nil, err } diff --git a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1/autoscaling.k8s.io_client.go b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1/autoscaling.k8s.io_client.go index 1ac1f8198c8a..f2932dc05145 100644 --- a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1/autoscaling.k8s.io_client.go +++ b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1/autoscaling.k8s.io_client.go @@ -50,9 +50,7 @@ func (c *AutoscalingV1Client) VerticalPodAutoscalerCheckpoints(namespace string) // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*AutoscalingV1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) httpClient, err := rest.HTTPClientFor(&config) if err != nil { return nil, err @@ -64,9 +62,7 @@ func NewForConfig(c *rest.Config) (*AutoscalingV1Client, error) { // Note the http client provided takes precedence over the configured transport values. func NewForConfigAndClient(c *rest.Config, h *http.Client) (*AutoscalingV1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err @@ -89,7 +85,7 @@ func New(c rest.Interface) *AutoscalingV1Client { return &AutoscalingV1Client{c} } -func setConfigDefaults(config *rest.Config) error { +func setConfigDefaults(config *rest.Config) { gv := autoscalingk8siov1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" @@ -98,8 +94,6 @@ func setConfigDefaults(config *rest.Config) error { if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - - return nil } // RESTClient returns a RESTClient that is used to communicate diff --git a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta1/autoscaling.k8s.io_client.go b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta1/autoscaling.k8s.io_client.go index 6b63e563d855..72ab4a4424e8 100644 --- a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta1/autoscaling.k8s.io_client.go +++ b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta1/autoscaling.k8s.io_client.go @@ -50,9 +50,7 @@ func (c *AutoscalingV1beta1Client) VerticalPodAutoscalerCheckpoints(namespace st // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*AutoscalingV1beta1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) httpClient, err := rest.HTTPClientFor(&config) if err != nil { return nil, err @@ -64,9 +62,7 @@ func NewForConfig(c *rest.Config) (*AutoscalingV1beta1Client, error) { // Note the http client provided takes precedence over the configured transport values. func NewForConfigAndClient(c *rest.Config, h *http.Client) (*AutoscalingV1beta1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err @@ -89,7 +85,7 @@ func New(c rest.Interface) *AutoscalingV1beta1Client { return &AutoscalingV1beta1Client{c} } -func setConfigDefaults(config *rest.Config) error { +func setConfigDefaults(config *rest.Config) { gv := autoscalingk8siov1beta1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" @@ -98,8 +94,6 @@ func setConfigDefaults(config *rest.Config) error { if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - - return nil } // RESTClient returns a RESTClient that is used to communicate diff --git a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta2/autoscaling.k8s.io_client.go b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta2/autoscaling.k8s.io_client.go index dd548845f28b..0ed2efcc49d5 100644 --- a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta2/autoscaling.k8s.io_client.go +++ b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta2/autoscaling.k8s.io_client.go @@ -50,9 +50,7 @@ func (c *AutoscalingV1beta2Client) VerticalPodAutoscalerCheckpoints(namespace st // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*AutoscalingV1beta2Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) httpClient, err := rest.HTTPClientFor(&config) if err != nil { return nil, err @@ -64,9 +62,7 @@ func NewForConfig(c *rest.Config) (*AutoscalingV1beta2Client, error) { // Note the http client provided takes precedence over the configured transport values. func NewForConfigAndClient(c *rest.Config, h *http.Client) (*AutoscalingV1beta2Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err @@ -89,7 +85,7 @@ func New(c rest.Interface) *AutoscalingV1beta2Client { return &AutoscalingV1beta2Client{c} } -func setConfigDefaults(config *rest.Config) error { +func setConfigDefaults(config *rest.Config) { gv := autoscalingk8siov1beta2.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" @@ -98,8 +94,6 @@ func setConfigDefaults(config *rest.Config) error { if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - - return nil } // RESTClient returns a RESTClient that is used to communicate diff --git a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/poc.autoscaling.k8s.io/v1alpha1/poc.autoscaling.k8s.io_client.go b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/poc.autoscaling.k8s.io/v1alpha1/poc.autoscaling.k8s.io_client.go index 23374e70aa5a..fc712073e8ec 100644 --- a/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/poc.autoscaling.k8s.io/v1alpha1/poc.autoscaling.k8s.io_client.go +++ b/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/poc.autoscaling.k8s.io/v1alpha1/poc.autoscaling.k8s.io_client.go @@ -50,9 +50,7 @@ func (c *PocV1alpha1Client) VerticalPodAutoscalerCheckpoints(namespace string) V // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*PocV1alpha1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) httpClient, err := rest.HTTPClientFor(&config) if err != nil { return nil, err @@ -64,9 +62,7 @@ func NewForConfig(c *rest.Config) (*PocV1alpha1Client, error) { // Note the http client provided takes precedence over the configured transport values. func NewForConfigAndClient(c *rest.Config, h *http.Client) (*PocV1alpha1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err @@ -89,7 +85,7 @@ func New(c rest.Interface) *PocV1alpha1Client { return &PocV1alpha1Client{c} } -func setConfigDefaults(config *rest.Config) error { +func setConfigDefaults(config *rest.Config) { gv := pocautoscalingk8siov1alpha1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" @@ -98,8 +94,6 @@ func setConfigDefaults(config *rest.Config) error { if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - - return nil } // RESTClient returns a RESTClient that is used to communicate diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscaler.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscaler.go index 854eee4935ad..6bb96d1ca431 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscaler.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscaler.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerInformer(client versioned.Interface, namesp if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1().VerticalPodAutoscalers(namespace).List(context.TODO(), options) + return client.AutoscalingV1().VerticalPodAutoscalers(namespace).List(context.Background(), options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1().VerticalPodAutoscalers(namespace).Watch(context.TODO(), options) + return client.AutoscalingV1().VerticalPodAutoscalers(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1().VerticalPodAutoscalers(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1().VerticalPodAutoscalers(namespace).Watch(ctx, options) }, }, &apisautoscalingk8siov1.VerticalPodAutoscaler{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscalercheckpoint.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscalercheckpoint.go index 5a6796aebf14..19b6e4f4a352 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscalercheckpoint.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1/verticalpodautoscalercheckpoint.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerCheckpointInformer(client versioned.Interfa if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1().VerticalPodAutoscalerCheckpoints(namespace).List(context.TODO(), options) + return client.AutoscalingV1().VerticalPodAutoscalerCheckpoints(namespace).List(context.Background(), options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.TODO(), options) + return client.AutoscalingV1().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1().VerticalPodAutoscalerCheckpoints(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1().VerticalPodAutoscalerCheckpoints(namespace).Watch(ctx, options) }, }, &apisautoscalingk8siov1.VerticalPodAutoscalerCheckpoint{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscaler.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscaler.go index 870dea95bb84..d94f9146feb1 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscaler.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscaler.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerInformer(client versioned.Interface, namesp if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta1().VerticalPodAutoscalers(namespace).List(context.TODO(), options) + return client.AutoscalingV1beta1().VerticalPodAutoscalers(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta1().VerticalPodAutoscalers(namespace).Watch(context.TODO(), options) + return client.AutoscalingV1beta1().VerticalPodAutoscalers(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta1().VerticalPodAutoscalers(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta1().VerticalPodAutoscalers(namespace).Watch(ctx, options) }, }, &apisautoscalingk8siov1beta1.VerticalPodAutoscaler{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscalercheckpoint.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscalercheckpoint.go index 73f3dbdbb167..82d4995ffe84 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscalercheckpoint.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta1/verticalpodautoscalercheckpoint.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerCheckpointInformer(client versioned.Interfa if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta1().VerticalPodAutoscalerCheckpoints(namespace).List(context.TODO(), options) + return client.AutoscalingV1beta1().VerticalPodAutoscalerCheckpoints(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta1().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.TODO(), options) + return client.AutoscalingV1beta1().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta1().VerticalPodAutoscalerCheckpoints(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta1().VerticalPodAutoscalerCheckpoints(namespace).Watch(ctx, options) }, }, &apisautoscalingk8siov1beta1.VerticalPodAutoscalerCheckpoint{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscaler.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscaler.go index 2a92542b09fe..e2b1c93ab068 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscaler.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscaler.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerInformer(client versioned.Interface, namesp if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta2().VerticalPodAutoscalers(namespace).List(context.TODO(), options) + return client.AutoscalingV1beta2().VerticalPodAutoscalers(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta2().VerticalPodAutoscalers(namespace).Watch(context.TODO(), options) + return client.AutoscalingV1beta2().VerticalPodAutoscalers(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta2().VerticalPodAutoscalers(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta2().VerticalPodAutoscalers(namespace).Watch(ctx, options) }, }, &apisautoscalingk8siov1beta2.VerticalPodAutoscaler{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscalercheckpoint.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscalercheckpoint.go index 3dcb75699a17..f2eb9c9ea7cc 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscalercheckpoint.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1beta2/verticalpodautoscalercheckpoint.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerCheckpointInformer(client versioned.Interfa if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta2().VerticalPodAutoscalerCheckpoints(namespace).List(context.TODO(), options) + return client.AutoscalingV1beta2().VerticalPodAutoscalerCheckpoints(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.AutoscalingV1beta2().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.TODO(), options) + return client.AutoscalingV1beta2().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta2().VerticalPodAutoscalerCheckpoints(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1beta2().VerticalPodAutoscalerCheckpoints(namespace).Watch(ctx, options) }, }, &apisautoscalingk8siov1beta2.VerticalPodAutoscalerCheckpoint{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscaler.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscaler.go index f3a8746ddad0..df5bd0dbd907 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscaler.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscaler.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerInformer(client versioned.Interface, namesp if tweakListOptions != nil { tweakListOptions(&options) } - return client.PocV1alpha1().VerticalPodAutoscalers(namespace).List(context.TODO(), options) + return client.PocV1alpha1().VerticalPodAutoscalers(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.PocV1alpha1().VerticalPodAutoscalers(namespace).Watch(context.TODO(), options) + return client.PocV1alpha1().VerticalPodAutoscalers(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.PocV1alpha1().VerticalPodAutoscalers(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.PocV1alpha1().VerticalPodAutoscalers(namespace).Watch(ctx, options) }, }, &apispocautoscalingk8siov1alpha1.VerticalPodAutoscaler{}, diff --git a/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscalercheckpoint.go b/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscalercheckpoint.go index 92012987fc95..eab9796af09b 100644 --- a/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscalercheckpoint.go +++ b/vertical-pod-autoscaler/pkg/client/informers/externalversions/poc.autoscaling.k8s.io/v1alpha1/verticalpodautoscalercheckpoint.go @@ -62,13 +62,25 @@ func NewFilteredVerticalPodAutoscalerCheckpointInformer(client versioned.Interfa if tweakListOptions != nil { tweakListOptions(&options) } - return client.PocV1alpha1().VerticalPodAutoscalerCheckpoints(namespace).List(context.TODO(), options) + return client.PocV1alpha1().VerticalPodAutoscalerCheckpoints(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.PocV1alpha1().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.TODO(), options) + return client.PocV1alpha1().VerticalPodAutoscalerCheckpoints(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.PocV1alpha1().VerticalPodAutoscalerCheckpoints(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.PocV1alpha1().VerticalPodAutoscalerCheckpoints(namespace).Watch(ctx, options) }, }, &apispocautoscalingk8siov1alpha1.VerticalPodAutoscalerCheckpoint{}, diff --git a/vertical-pod-autoscaler/pkg/features/features.go b/vertical-pod-autoscaler/pkg/features/features.go index 2c34ac400178..5f92b6013c2b 100644 --- a/vertical-pod-autoscaler/pkg/features/features.go +++ b/vertical-pod-autoscaler/pkg/features/features.go @@ -48,6 +48,15 @@ const ( // InPlaceOrRecreate enables the InPlaceOrRecreate update mode to be used. // Requires KEP-1287 InPlacePodVerticalScaling feature-gate to be enabled on the cluster. InPlaceOrRecreate featuregate.Feature = "InPlaceOrRecreate" + + // alpha: v1.5.0 + // components: recommender, updater + + // PerVPAConfig enables the ability to specify component-specific configuration + // parameters at the individual VPA object level. This allows for different + // optimization strategies to be applied to different workloads within the + // same cluster. + PerVPAConfig featuregate.Feature = "PerVPAConfig" ) // MutableFeatureGate is a mutable, versioned, global FeatureGate. diff --git a/vertical-pod-autoscaler/pkg/features/versioned_features.go b/vertical-pod-autoscaler/pkg/features/versioned_features.go index e623061fffd9..19f2feb270c0 100644 --- a/vertical-pod-autoscaler/pkg/features/versioned_features.go +++ b/vertical-pod-autoscaler/pkg/features/versioned_features.go @@ -31,4 +31,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned {Version: version.MustParse("1.4"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.5"), Default: true, PreRelease: featuregate.Beta}, }, + PerVPAConfig: { + {Version: version.MustParse("1.5"), Default: false, PreRelease: featuregate.Alpha}, + }, } diff --git a/vertical-pod-autoscaler/pkg/recommender/main.go b/vertical-pod-autoscaler/pkg/recommender/main.go index 4f0f2f997eaf..530de47abb63 100644 --- a/vertical-pod-autoscaler/pkg/recommender/main.go +++ b/vertical-pod-autoscaler/pkg/recommender/main.go @@ -103,8 +103,8 @@ var ( memoryAggregationIntervalCount = flag.Int64("memory-aggregation-interval-count", model.DefaultMemoryAggregationIntervalCount, `The number of consecutive memory-aggregation-intervals which make up the MemoryAggregationWindowLength which in turn is the period for memory usage aggregation by VPA. In other words, MemoryAggregationWindowLength = memory-aggregation-interval * memory-aggregation-interval-count.`) memoryHistogramDecayHalfLife = flag.Duration("memory-histogram-decay-half-life", model.DefaultMemoryHistogramDecayHalfLife, `The amount of time it takes a historical memory usage sample to lose half of its weight. In other words, a fresh usage sample is twice as 'important' as one with age equal to the half life period.`) cpuHistogramDecayHalfLife = flag.Duration("cpu-histogram-decay-half-life", model.DefaultCPUHistogramDecayHalfLife, `The amount of time it takes a historical CPU usage sample to lose half of its weight.`) - oomBumpUpRatio = flag.Float64("oom-bump-up-ratio", model.DefaultOOMBumpUpRatio, `The memory bump up ratio when OOM occurred, default is 1.2.`) - oomMinBumpUp = flag.Float64("oom-min-bump-up-bytes", model.DefaultOOMMinBumpUp, `The minimal increase of memory when OOM occurred in bytes, default is 100 * 1024 * 1024`) + oomBumpUpRatio = flag.Float64("oom-bump-up-ratio", model.DefaultOOMBumpUpRatio, `Default memory bump up ratio when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 1.2.`) + oomMinBumpUp = flag.Float64("oom-min-bump-up-bytes", model.DefaultOOMMinBumpUp, `Default minimal increase of memory (in bytes) when OOM occurs. This value applies to all VPAs unless overridden in the VPA spec. Default is 100 * 1024 * 1024 (100Mi).`) ) // Post processors flags diff --git a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go index c18e860f4ff7..083322b7426c 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go @@ -40,9 +40,12 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" ) @@ -81,6 +84,10 @@ type ContainerStateAggregator interface { // GetUpdateMode returns the update mode of VPA controlling this aggregator, // nil if aggregator is not autoscaled. GetUpdateMode() *vpa_types.UpdateMode + // GetOOMBumpUpRatio returns the OOM bump up ratio for this container + GetOOMBumpUpRatio() float64 + // GetOOMMinBumpUp returns the minimum OOM bump up value for this container + GetOOMMinBumpUp() float64 } // AggregateContainerState holds input signals aggregated from a set of containers. @@ -109,6 +116,8 @@ type AggregateContainerState struct { IsUnderVPA bool UpdateMode *vpa_types.UpdateMode ScalingMode *vpa_types.ContainerScalingMode + OOMBumpUpRatio float64 + OOMMinBumpUp float64 ControlledResources *[]ResourceName } @@ -143,6 +152,16 @@ func (a *AggregateContainerState) GetControlledResources() []ResourceName { return DefaultControlledResources } +// GetOOMBumpUpRatio returns the ratio by which to increase the memory recommendation in case of OOM +func (a *AggregateContainerState) GetOOMBumpUpRatio() float64 { + return a.OOMBumpUpRatio +} + +// GetOOMMinBumpUp returns the minimum absolute increase in memory recommendation in case of OOM +func (a *AggregateContainerState) GetOOMMinBumpUp() float64 { + return a.OOMMinBumpUp +} + // MarkNotAutoscaled registers that this container state is not controlled by // a VPA object. func (a *AggregateContainerState) MarkNotAutoscaled() { @@ -175,6 +194,8 @@ func NewAggregateContainerState() *AggregateContainerState { AggregateCPUUsage: util.NewDecayingHistogram(config.CPUHistogramOptions, config.CPUHistogramDecayHalfLife), AggregateMemoryPeaks: util.NewDecayingHistogram(config.MemoryHistogramOptions, config.MemoryHistogramDecayHalfLife), CreationTime: time.Now(), + OOMBumpUpRatio: config.OOMBumpUpRatio, + OOMMinBumpUp: config.OOMMinBumpUp, } } @@ -270,6 +291,13 @@ func (a *AggregateContainerState) isEmpty() bool { return a.TotalSamplesCount == 0 } +func (a *AggregateContainerState) convertQuantityToFloat64(quantity *resource.Quantity) float64 { + if quantity == nil { + return 0.0 + } + return float64(quantity.MilliValue()) / 1000.0 +} + // UpdateFromPolicy updates container state scaling mode and controlled resources based on resource // policy of the VPA object. func (a *AggregateContainerState) UpdateFromPolicy(resourcePolicy *vpa_types.ContainerResourcePolicy) { @@ -283,6 +311,25 @@ func (a *AggregateContainerState) UpdateFromPolicy(resourcePolicy *vpa_types.Con if resourcePolicy != nil && resourcePolicy.ControlledResources != nil { a.ControlledResources = ResourceNamesApiToModel(*resourcePolicy.ControlledResources) } + + // Per VPA components - feature flag "PerVPAConfig" must be enabled + if resourcePolicy != nil { + if resourcePolicy.OOMBumpUpRatio != nil { + if features.Enabled(features.PerVPAConfig) { + a.OOMBumpUpRatio = a.convertQuantityToFloat64(resourcePolicy.OOMBumpUpRatio) + } else { + klog.InfoS("OOMBumpUpRatio is set but PerVPAConfig feature gate is disabled, falling back to default value") + } + } + + if resourcePolicy.OOMMinBumpUp != nil { + if features.Enabled(features.PerVPAConfig) { + a.OOMMinBumpUp = a.convertQuantityToFloat64(resourcePolicy.OOMMinBumpUp) + } else { + klog.InfoS("OOMMinBumpUp is set but PerVPAConfig feature gate is disabled, falling back to default value") + } + } + } } // AggregateStateByContainerName takes a set of AggregateContainerStates and merge them @@ -351,3 +398,17 @@ func (p *ContainerStateAggregatorProxy) GetScalingMode() *vpa_types.ContainerSca aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) return aggregator.GetScalingMode() } + +// GetOOMMinBumpUp returns the minimum amount to bump up resources when OOM is detected. +// This implementation returns 0 to satisfy the interface requirement. +func (p *ContainerStateAggregatorProxy) GetOOMMinBumpUp() float64 { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + return aggregator.GetOOMMinBumpUp() +} + +// GetOOMBumpUpRatio returns the ratio to increase resources when OOM is detected. +// This implementation returns 0 to satisfy the interface requirement. +func (p *ContainerStateAggregatorProxy) GetOOMBumpUpRatio() float64 { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + return aggregator.GetOOMBumpUpRatio() +} diff --git a/vertical-pod-autoscaler/pkg/recommender/model/container.go b/vertical-pod-autoscaler/pkg/recommender/model/container.go index cbb1052432fe..6f0d64b65f6c 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/container.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/container.go @@ -125,6 +125,18 @@ func (container *ContainerState) GetMaxMemoryPeak() ResourceAmount { return ResourceAmountMax(container.memoryPeak, container.oomPeak) } +// GetOOMBumpUpRatio returns the ratio to increase resources when OOM is detected. +// It delegates to the aggregator's implementation. +func (container *ContainerState) GetOOMBumpUpRatio() float64 { + return container.aggregator.GetOOMBumpUpRatio() +} + +// GetOOMMinBumpUp returns the minimum amount to bump up resources when OOM is detected. +// It delegates to the aggregator's implementation. +func (container *ContainerState) GetOOMMinBumpUp() float64 { + return container.aggregator.GetOOMMinBumpUp() +} + func (container *ContainerState) addMemorySample(sample *ContainerUsageSample, isOOM bool) bool { ts := sample.MeasureStart // We always process OOM samples. @@ -183,14 +195,16 @@ func (container *ContainerState) addMemorySample(sample *ContainerUsageSample, i // RecordOOM adds info regarding OOM event in the model as an artificial memory sample. func (container *ContainerState) RecordOOM(timestamp time.Time, requestedMemory ResourceAmount) error { // Discard old OOM - if timestamp.Before(container.WindowEnd.Add(-1 * GetAggregationsConfig().MemoryAggregationInterval)) { + config := GetAggregationsConfig() + // TODO(omerap12): remove MemoryAggregationInterval to per-container configuration as well + if timestamp.Before(container.WindowEnd.Add(-1 * config.MemoryAggregationInterval)) { return fmt.Errorf("OOM event will be discarded - it is too old (%v)", timestamp) } // Get max of the request and the recent usage-based memory peak. // Omitting oomPeak here to protect against recommendation running too high on subsequent OOMs. memoryUsed := ResourceAmountMax(requestedMemory, container.memoryPeak) - memoryNeeded := ResourceAmountMax(memoryUsed+MemoryAmountFromBytes(GetAggregationsConfig().OOMMinBumpUp), - ScaleResource(memoryUsed, GetAggregationsConfig().OOMBumpUpRatio)) + memoryNeeded := ResourceAmountMax(memoryUsed+MemoryAmountFromBytes(container.GetOOMMinBumpUp()), + ScaleResource(memoryUsed, container.GetOOMBumpUpRatio())) oomMemorySample := ContainerUsageSample{ MeasureStart: timestamp, diff --git a/vertical-pod-autoscaler/pkg/recommender/model/container_test.go b/vertical-pod-autoscaler/pkg/recommender/model/container_test.go index 59a004d303db..45203f72d35a 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/container_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/container_test.go @@ -61,6 +61,8 @@ func newContainerTest() ContainerTest { aggregateContainerState := &AggregateContainerState{ AggregateCPUUsage: mockCPUHistogram, AggregateMemoryPeaks: mockMemoryHistogram, + OOMBumpUpRatio: 1.2, // Default value, can be adjusted as needed + OOMMinBumpUp: 1.048576e+08, // Default value (100Mi), can be adjusted as needed } container := &ContainerState{ Request: TestRequest, diff --git a/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go b/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go index 6bb3279baff9..5a32bc0a2069 100644 --- a/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go +++ b/vertical-pod-autoscaler/pkg/utils/test/test_vpa.go @@ -21,6 +21,7 @@ import ( autoscaling "k8s.io/api/autoscaling/v1" core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" meta "k8s.io/apimachinery/pkg/apis/meta/v1" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" @@ -47,6 +48,8 @@ type VerticalPodAutoscalerBuilder interface { WithGroupVersion(gv meta.GroupVersion) VerticalPodAutoscalerBuilder WithEvictionRequirements([]*vpa_types.EvictionRequirement) VerticalPodAutoscalerBuilder WithMinReplicas(minReplicas *int32) VerticalPodAutoscalerBuilder + WithOOMBumpUpRatio(ratio *resource.Quantity) VerticalPodAutoscalerBuilder + WithOOMMinBumpUp(minBumpUp *resource.Quantity) VerticalPodAutoscalerBuilder AppendCondition(conditionType vpa_types.VerticalPodAutoscalerConditionType, status core.ConditionStatus, reason, message string, lastTransitionTime time.Time) VerticalPodAutoscalerBuilder AppendRecommendation(vpa_types.RecommendedContainerResources) VerticalPodAutoscalerBuilder @@ -87,6 +90,8 @@ type verticalPodAutoscalerBuilder struct { targetRef *autoscaling.CrossVersionObjectReference appendedRecommendations []vpa_types.RecommendedContainerResources recommender string + oomBumpUpRatio *resource.Quantity + oomMinBumpUp *resource.Quantity } func (b *verticalPodAutoscalerBuilder) WithName(vpaName string) VerticalPodAutoscalerBuilder { @@ -214,6 +219,18 @@ func (b *verticalPodAutoscalerBuilder) WithMinReplicas(minReplicas *int32) Verti return &c } +func (b *verticalPodAutoscalerBuilder) WithOOMBumpUpRatio(ratio *resource.Quantity) VerticalPodAutoscalerBuilder { + c := *b + c.oomBumpUpRatio = ratio + return &c +} + +func (b *verticalPodAutoscalerBuilder) WithOOMMinBumpUp(minBumpUp *resource.Quantity) VerticalPodAutoscalerBuilder { + c := *b + c.oomMinBumpUp = minBumpUp + return &c +} + func (b *verticalPodAutoscalerBuilder) AppendCondition(conditionType vpa_types.VerticalPodAutoscalerConditionType, status core.ConditionStatus, reason, message string, lastTransitionTime time.Time) VerticalPodAutoscalerBuilder { c := *b @@ -250,6 +267,8 @@ func (b *verticalPodAutoscalerBuilder) Get() *vpa_types.VerticalPodAutoscaler { MaxAllowed: b.maxAllowed[containerName], ControlledValues: b.controlledValues[containerName], Mode: &scalingModeAuto, + OOMBumpUpRatio: b.oomBumpUpRatio, + OOMMinBumpUp: b.oomMinBumpUp, } if scalingMode, ok := b.scalingMode[containerName]; ok { containerResourcePolicy.Mode = scalingMode