added --reserved-cpus kubelet command option

Author: jianzzha

cmd/kubeadm/.import-restrictions

@@ -64,6 +64,7 @@
 				"k8s.io/kubernetes/pkg/features",
 				"k8s.io/kubernetes/pkg/fieldpath",
 				"k8s.io/kubernetes/pkg/kubelet/apis",
+				"k8s.io/kubernetes/pkg/kubelet/cm/cpuset",
 				"k8s.io/kubernetes/pkg/kubelet/qos",
 				"k8s.io/kubernetes/pkg/kubelet/types",
 				"k8s.io/kubernetes/pkg/master/ports",

cmd/kubelet/app/BUILD

@@ -57,6 +57,7 @@ go_library(
         "//pkg/kubelet/certificate:go_default_library",
         "//pkg/kubelet/certificate/bootstrap:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
+        "//pkg/kubelet/cm/cpuset:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/dockershim:go_default_library",

cmd/kubelet/app/options/options.go

@@ -553,7 +553,7 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig
 	fs.Var(cliflag.NewMapStringString(&c.EvictionMinimumReclaim), "eviction-minimum-reclaim", "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.")
 	fs.Int32Var(&c.PodsPerCore, "pods-per-core", c.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.")
 	fs.BoolVar(&c.ProtectKernelDefaults, "protect-kernel-defaults", c.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.")
-
+	fs.StringVar(&c.ReservedSystemCPUs, "reserved-cpus", c.ReservedSystemCPUs, "A comma-separated list of CPUs or CPU ranges that are reserved for system and kubernetes usage. This specific list will supersede cpu counts in --system-reserved and --kube-reserved.")
 	// Node Allocatable Flags
 	fs.Var(cliflag.NewMapStringString(&c.SystemReserved), "system-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi,ephemeral-storage=1Gi) pairs that describe resources reserved for non-kubernetes components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]")
 	fs.Var(cliflag.NewMapStringString(&c.KubeReserved), "kube-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi,ephemeral-storage=1Gi) pairs that describe resources reserved for kubernetes system components. Currently cpu, memory and local ephemeral storage for root file system are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]")

cmd/kubelet/app/server.go

@@ -77,6 +77,7 @@ import (
 	kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/dockershim"
@@ -636,6 +637,48 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 			klog.Info("--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /")
 			s.CgroupRoot = "/"
 		}
+
+		var reservedSystemCPUs cpuset.CPUSet
+		var errParse error
+		if s.ReservedSystemCPUs != "" {
+			reservedSystemCPUs, errParse = cpuset.Parse(s.ReservedSystemCPUs)
+			if errParse != nil {
+				// invalid cpu list is provided, set reservedSystemCPUs to empty, so it won't overwrite kubeReserved/systemReserved
+				klog.Infof("Invalid ReservedSystemCPUs \"%s\"", s.ReservedSystemCPUs)
+				return errParse
+			}
+			// is it safe do use CAdvisor here ??
+			machineInfo, err := kubeDeps.CAdvisorInterface.MachineInfo()
+			if err != nil {
+				// if can't use CAdvisor here, fall back to non-explicit cpu list behavor
+				klog.Warning("Failed to get MachineInfo, set reservedSystemCPUs to empty")
+				reservedSystemCPUs = cpuset.NewCPUSet()
+			} else {
+				reservedList := reservedSystemCPUs.ToSlice()
+				first := reservedList[0]
+				last := reservedList[len(reservedList)-1]
+				if first < 0 || last >= machineInfo.NumCores {
+					// the specified cpuset is outside of the range of what the machine has
+					klog.Infof("Invalid cpuset specified by --reserved-cpus")
+					return fmt.Errorf("Invalid cpuset %q specified by --reserved-cpus", s.ReservedSystemCPUs)
+				}
+			}
+		} else {
+			reservedSystemCPUs = cpuset.NewCPUSet()
+		}
+
+		if reservedSystemCPUs.Size() > 0 {
+			// at cmd option valication phase it is tested either --system-reserved-cgroup or --kube-reserved-cgroup is specified, so overwrite should be ok
+			klog.Infof("Option --reserved-cpus is specified, it will overwrite the cpu setting in KubeReserved=\"%v\", SystemReserved=\"%v\".", s.KubeReserved, s.SystemReserved)
+			if s.KubeReserved != nil {
+				delete(s.KubeReserved, "cpu")
+			}
+			if s.SystemReserved == nil {
+				s.SystemReserved = make(map[string]string)
+			}
+			s.SystemReserved["cpu"] = strconv.Itoa(reservedSystemCPUs.Size())
+			klog.Infof("After cpu setting is overwritten, KubeReserved=\"%v\", SystemReserved=\"%v\"", s.KubeReserved, s.SystemReserved)
+		}
 		kubeReserved, err := parseResourceList(s.KubeReserved)
 		if err != nil {
 			return err
@@ -678,6 +721,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 					EnforceNodeAllocatable:   sets.NewString(s.EnforceNodeAllocatable...),
 					KubeReserved:             kubeReserved,
 					SystemReserved:           systemReserved,
+					ReservedSystemCPUs:       reservedSystemCPUs,
 					HardEvictionThresholds:   hardEvictionThresholds,
 				},
 				QOSReserved:                           *experimentalQOSReserved,

pkg/kubelet/apis/config/helpers_test.go

@@ -212,6 +212,7 @@ var (
 		"ReadOnlyPort",
 		"RegistryBurst",
 		"RegistryPullQPS",
+		"ReservedSystemCPUs",
 		"RuntimeRequestTimeout.Duration",
 		"SerializeImagePulls",
 		"StreamingConnectionIdleTimeout.Duration",

pkg/kubelet/apis/config/types.go

@@ -331,6 +331,10 @@ type KubeletConfiguration struct {
 	// This flag accepts a list of options. Acceptable options are `pods`, `system-reserved` & `kube-reserved`.
 	// Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information.
 	EnforceNodeAllocatable []string
+	// This option specifies the cpu list reserved for the host level system threads and kubernetes related threads.
+	// This provide a "static" CPU list rather than the "dynamic" list by system-reserved and kube-reserved.
+	// This option overwrites CPUs provided by system-reserved and kube-reserved.
+	ReservedSystemCPUs string
 }
 
 type KubeletAuthorizationMode string

pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go

@@ -17,6 +17,7 @@ go_library(
     deps = [
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis/config:go_default_library",
+        "//pkg/kubelet/cm/cpuset:go_default_library",
         "//pkg/kubelet/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",

pkg/kubelet/apis/config/validation/BUILD

@@ -25,6 +25,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/features"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
@@ -142,6 +143,15 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error
 		allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for HairpinMode (--hairpin-mode). Valid options are %q, %q or %q",
 			kc.HairpinMode, kubeletconfig.HairpinNone, kubeletconfig.HairpinVeth, kubeletconfig.PromiscuousBridge))
 	}
+	if kc.ReservedSystemCPUs != "" {
+		// --reserved-cpus does not support --system-reserved-cgroup or --kube-reserved-cgroup
+		if kc.SystemReservedCgroup != "" || kc.KubeReservedCgroup != "" {
+			allErrors = append(allErrors, fmt.Errorf("can't use --reserved-cpus with --system-reserved-cgroup or --kube-reserved-cgroup"))
+		}
+		if _, err := cpuset.Parse(kc.ReservedSystemCPUs); err != nil {
+			allErrors = append(allErrors, fmt.Errorf("unable to parse --reserved-cpus, error: %v", err))
+		}
+	}
 
 	if err := validateKubeletOSConfiguration(kc); err != nil {
 		allErrors = append(allErrors, err)

pkg/kubelet/apis/config/validation/validation.go

@@ -26,7 +26,7 @@ import (
 )
 
 func TestValidateKubeletConfiguration(t *testing.T) {
-	successCase := &kubeletconfig.KubeletConfiguration{
+	successCase1 := &kubeletconfig.KubeletConfiguration{
 		CgroupsPerQOS:               true,
 		EnforceNodeAllocatable:      []string{"pods", "system-reserved", "kube-reserved"},
 		SystemReservedCgroup:        "/system.slice",
@@ -55,11 +55,45 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 		CPUCFSQuotaPeriod:           metav1.Duration{Duration: 100 * time.Millisecond},
 		TopologyManagerPolicy:       "none",
 	}
-	if allErrors := ValidateKubeletConfiguration(successCase); allErrors != nil {
+	if allErrors := ValidateKubeletConfiguration(successCase1); allErrors != nil {
 		t.Errorf("expect no errors, got %v", allErrors)
 	}
 
-	errorCase := &kubeletconfig.KubeletConfiguration{
+	successCase2 := &kubeletconfig.KubeletConfiguration{
+		CgroupsPerQOS:               true,
+		EnforceNodeAllocatable:      []string{"pods"},
+		SystemReservedCgroup:        "",
+		KubeReservedCgroup:          "",
+		SystemCgroups:               "",
+		CgroupRoot:                  "",
+		EventBurst:                  10,
+		EventRecordQPS:              5,
+		HealthzPort:                 10248,
+		ImageGCHighThresholdPercent: 85,
+		ImageGCLowThresholdPercent:  80,
+		IPTablesDropBit:             15,
+		IPTablesMasqueradeBit:       14,
+		KubeAPIBurst:                10,
+		KubeAPIQPS:                  5,
+		MaxOpenFiles:                1000000,
+		MaxPods:                     110,
+		OOMScoreAdj:                 -999,
+		PodsPerCore:                 100,
+		Port:                        65535,
+		ReadOnlyPort:                0,
+		RegistryBurst:               10,
+		RegistryPullQPS:             5,
+		HairpinMode:                 kubeletconfig.PromiscuousBridge,
+		NodeLeaseDurationSeconds:    1,
+		CPUCFSQuotaPeriod:           metav1.Duration{Duration: 100 * time.Millisecond},
+		TopologyManagerPolicy:       "none",
+		ReservedSystemCPUs:          "0-3",
+	}
+	if allErrors := ValidateKubeletConfiguration(successCase2); allErrors != nil {
+		t.Errorf("expect no errors, got %v", allErrors)
+	}
+
+	errorCase1 := &kubeletconfig.KubeletConfiguration{
 		CgroupsPerQOS:               false,
 		EnforceNodeAllocatable:      []string{"pods", "system-reserved", "kube-reserved", "illegal-key"},
 		SystemCgroups:               "/",
@@ -86,8 +120,43 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 		CPUCFSQuotaPeriod:           metav1.Duration{Duration: 0},
 		TopologyManagerPolicy:       "",
 	}
-	const numErrs = 26
-	if allErrors := ValidateKubeletConfiguration(errorCase); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrs {
-		t.Errorf("expect %d errors, got %v", numErrs, len(allErrors.(utilerrors.Aggregate).Errors()))
+	const numErrsErrorCase1 = 26
+	if allErrors := ValidateKubeletConfiguration(errorCase1); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrsErrorCase1 {
+		t.Errorf("expect %d errors, got %v", numErrsErrorCase1, len(allErrors.(utilerrors.Aggregate).Errors()))
+	}
+
+	errorCase2 := &kubeletconfig.KubeletConfiguration{
+		CgroupsPerQOS:               true,
+		EnforceNodeAllocatable:      []string{"pods", "system-reserved", "kube-reserved"},
+		SystemReservedCgroup:        "/system.slice",
+		KubeReservedCgroup:          "/kubelet.service",
+		SystemCgroups:               "",
+		CgroupRoot:                  "",
+		EventBurst:                  10,
+		EventRecordQPS:              5,
+		HealthzPort:                 10248,
+		ImageGCHighThresholdPercent: 85,
+		ImageGCLowThresholdPercent:  80,
+		IPTablesDropBit:             15,
+		IPTablesMasqueradeBit:       14,
+		KubeAPIBurst:                10,
+		KubeAPIQPS:                  5,
+		MaxOpenFiles:                1000000,
+		MaxPods:                     110,
+		OOMScoreAdj:                 -999,
+		PodsPerCore:                 100,
+		Port:                        65535,
+		ReadOnlyPort:                0,
+		RegistryBurst:               10,
+		RegistryPullQPS:             5,
+		HairpinMode:                 kubeletconfig.PromiscuousBridge,
+		NodeLeaseDurationSeconds:    1,
+		CPUCFSQuotaPeriod:           metav1.Duration{Duration: 100 * time.Millisecond},
+		TopologyManagerPolicy:       "none",
+		ReservedSystemCPUs:          "0-3",
+	}
+	const numErrsErrorCase2 = 1
+	if allErrors := ValidateKubeletConfiguration(errorCase2); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrsErrorCase2 {
+		t.Errorf("expect %d errors, got %v", numErrsErrorCase2, len(allErrors.(utilerrors.Aggregate).Errors()))
 	}
 }