Merge pull request kubernetes#126096 from utam0k/support-disabling-oom-group-kill

kubelet: new kubelet config option for disabling group oom kill

Author: k8s-ci-robot

cmd/kubelet/app/server.go

@@ -606,7 +606,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
 
 	// Warn if MemoryQoS enabled with cgroups v1
 	if utilfeature.DefaultFeatureGate.Enabled(features.MemoryQoS) &&
-		!isCgroup2UnifiedMode() {
+		!kubeletutil.IsCgroup2UnifiedMode() {
 		klog.InfoS("Warning: MemoryQoS feature only works with cgroups v2 on Linux, but enabled with cgroups v1")
 	}
 	// Obtain Kubelet Lock File
@@ -831,7 +831,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
 				s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions)
 		}
 		if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) {
-			if !isCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == kubelettypes.LimitedSwap {
+			if !kubeletutil.IsCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == kubelettypes.LimitedSwap {
 				// This feature is not supported for cgroupv1 so we are failing early.
 				return fmt.Errorf("swap feature is enabled and LimitedSwap but it is only supported with cgroupv2")
 			}

cmd/kubelet/app/server_linux.go

@@ -19,8 +19,6 @@ package app
 import (
 	"k8s.io/klog/v2"
 	"k8s.io/utils/inotify"
-
-	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
 )
 
 func watchForLockfileContention(path string, done chan struct{}) error {
@@ -46,7 +44,3 @@ func watchForLockfileContention(path string, done chan struct{}) error {
 	}()
 	return nil
 }
-
-func isCgroup2UnifiedMode() bool {
-	return libcontainercgroups.IsCgroup2UnifiedMode()
-}

pkg/generated/openapi/zz_generated.openapi.go

@@ -73,12 +73,12 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.NodeStatusReportFrequency = metav1.Duration{Duration: time.Minute}
 			obj.NodeLeaseDurationSeconds = 40
 			obj.CPUManagerPolicy = "none"
-			obj.CPUManagerPolicyOptions = make(map[string]string)
+			obj.CPUManagerPolicyOptions = nil
 			obj.CPUManagerReconcilePeriod = obj.NodeStatusUpdateFrequency
 			obj.NodeStatusMaxImages = 50
 			obj.TopologyManagerPolicy = kubeletconfig.NoneTopologyManagerPolicy
 			obj.TopologyManagerScope = kubeletconfig.ContainerTopologyManagerScope
-			obj.TopologyManagerPolicyOptions = make(map[string]string)
+			obj.TopologyManagerPolicyOptions = nil
 			obj.QOSReserved = map[string]string{
 				"memory": "50%",
 			}
@@ -104,13 +104,14 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.CgroupsPerQOS = true
 			obj.CgroupDriver = "cgroupfs"
 			obj.EnforceNodeAllocatable = kubeletconfigv1beta1.DefaultNodeAllocatableEnforcement
-			obj.StaticPodURLHeader = make(map[string][]string)
+			obj.StaticPodURLHeader = nil
+			obj.SingleProcessOOMKill = ptr.To(false)
 			obj.ContainerLogMaxFiles = 5
 			obj.ContainerLogMaxSize = "10Mi"
 			obj.ContainerLogMaxWorkers = 1
 			obj.ContainerLogMonitorInterval = metav1.Duration{Duration: 10 * time.Second}
 			obj.ConfigMapAndSecretChangeDetectionStrategy = "Watch"
-			obj.AllowedUnsafeSysctls = []string{}
+			obj.AllowedUnsafeSysctls = nil
 			obj.VolumePluginDir = kubeletconfigv1beta1.DefaultVolumePluginDir
 			obj.ContainerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"
 

pkg/kubelet/apis/config/fuzzer/fuzzer.go

@@ -233,6 +233,7 @@ var (
 		"Logging.Options.Text.OutputRoutingOptions.SplitStream",
 		"Logging.VModule[*].FilePattern",
 		"Logging.VModule[*].Verbosity",
+		"SingleProcessOOMKill",
 		"Logging.Verbosity",
 		"TLSCipherSuites[*]",
 		"TLSMinVersion",

pkg/kubelet/apis/config/helpers_test.go

@@ -229,6 +229,10 @@ type KubeletConfiguration struct {
 	CgroupsPerQOS bool
 	// driver that the kubelet uses to manipulate cgroups on the host (cgroupfs or systemd)
 	CgroupDriver string
+	// SingleProcessOOMKill, if true, will prevent the `memory.oom.group` flag from being set for container
+	// cgroups in cgroups v2. This causes processes in the container to be OOM killed individually instead of as
+	// a group. It means that if true, the behavior aligns with the behavior of cgroups v1.
+	SingleProcessOOMKill *bool
 	// CPUManagerPolicy is the name of the policy to use.
 	// Requires the CPUManager feature gate to be enabled.
 	CPUManagerPolicy string

pkg/kubelet/apis/config/types.go

@@ -34,7 +34,6 @@ import (
 )
 
 func TestSetDefaultsKubeletConfiguration(t *testing.T) {
-
 	tests := []struct {
 		name     string
 		config   *v1beta1.KubeletConfiguration
@@ -130,6 +129,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(true),
 				LocalStorageCapacityIsolation: ptr.To(true),
 				PodLogsDir:                    DefaultPodLogsDir,
+				SingleProcessOOMKill:          nil,
 			},
 		},
 		{
@@ -261,6 +261,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                    ptr.To(false),
 				LocalStorageCapacityIsolation:   ptr.To(false),
 				PodLogsDir:                      "",
+				SingleProcessOOMKill:            ptr.To(false),
 			},
 			&v1beta1.KubeletConfiguration{
 				EnableServer:       ptr.To(false),
@@ -363,6 +364,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(false),
 				LocalStorageCapacityIsolation: ptr.To(false),
 				PodLogsDir:                    DefaultPodLogsDir,
+				SingleProcessOOMKill:          ptr.To(false),
 			},
 		},
 		{
@@ -516,6 +518,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(true),
 				LocalStorageCapacityIsolation: ptr.To(true),
 				PodLogsDir:                    "/custom/path",
+				SingleProcessOOMKill:          ptr.To(true),
 			},
 			&v1beta1.KubeletConfiguration{
 				EnableServer:       ptr.To(true),
@@ -666,6 +669,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(true),
 				LocalStorageCapacityIsolation: ptr.To(true),
 				PodLogsDir:                    "/custom/path",
+				SingleProcessOOMKill:          ptr.To(true),
 			},
 		},
 		{
@@ -759,6 +763,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(true),
 				LocalStorageCapacityIsolation: ptr.To(true),
 				PodLogsDir:                    DefaultPodLogsDir,
+				SingleProcessOOMKill:          nil,
 			},
 		},
 		{
@@ -852,6 +857,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(true),
 				LocalStorageCapacityIsolation: ptr.To(true),
 				PodLogsDir:                    DefaultPodLogsDir,
+				SingleProcessOOMKill:          nil,
 			},
 		},
 		{
@@ -945,6 +951,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				RegisterNode:                  ptr.To(true),
 				LocalStorageCapacityIsolation: ptr.To(true),
 				PodLogsDir:                    DefaultPodLogsDir,
+				SingleProcessOOMKill:          nil,
 			},
 		},
 	}

pkg/kubelet/apis/config/v1beta1/defaults_test.go

@@ -24,13 +24,19 @@ import (
 
 	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	"k8s.io/utils/ptr"
 )
 
 // validateKubeletOSConfiguration validates os specific kubelet configuration and returns an error if it is invalid.
 func validateKubeletOSConfiguration(kc *kubeletconfig.KubeletConfiguration) error {
-	if kc.FailCgroupV1 && !libcontainercgroups.IsCgroup2UnifiedMode() {
+	isCgroup1 := !libcontainercgroups.IsCgroup2UnifiedMode()
+	if kc.FailCgroupV1 && isCgroup1 {
 		return fmt.Errorf("kubelet is configured to not run on a host using cgroup v1. cgroup v1 support is in maintenance mode")
 	}
 
+	if isCgroup1 && kc.SingleProcessOOMKill != nil && !ptr.Deref(kc.SingleProcessOOMKill, true) {
+		return fmt.Errorf("invalid configuration: singleProcessOOMKill must not be explicitly set to false when using cgroup v1")
+	}
+
 	return nil
 }

pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go

@@ -20,10 +20,16 @@ limitations under the License.
 package validation
 
 import (
+	"fmt"
+
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 )
 
 // validateKubeletOSConfiguration validates os specific kubelet configuration and returns an error if it is invalid.
 func validateKubeletOSConfiguration(kc *kubeletconfig.KubeletConfiguration) error {
+	if kc.SingleProcessOOMKill != nil {
+		return fmt.Errorf("invalid configuration: singleProcessOOMKill is only supported on linux")
+	}
+
 	return nil
 }