Skip to content

Commit 5ea57fb

Browse files
ndixitandixita
committed
cgroup configuration changes:
1. Pod cgrooup configured to use resources from pod spec if feature is enabled and resources are set at pod-level 2. Container cgroup limits defaulted to pod-level limits is container limits are not set
1 parent 26f11c4 commit 5ea57fb

File tree

6 files changed

+214
-18
lines changed

6 files changed

+214
-18
lines changed

‎pkg/kubelet/cm/helpers_linux.go‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,16 +118,20 @@ func HugePageLimits(resourceList v1.ResourceList) map[int64]int64 {
118118

119119
// ResourceConfigForPod takes the input pod and outputs the cgroup resource config.
120120
func ResourceConfigForPod(allocatedPod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64, enforceMemoryQoS bool) *ResourceConfig {
121+
podLevelResourcesEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources)
122+
// sum requests and limits.
121123
reqs := resource.PodRequests(allocatedPod, resource.PodResourcesOptions{
122-
// pod is already configured to the allocated resources, and we explicitly don't want to use
123-
// the actual resources if we're instantiating a resize.
124-
UseStatusResources: false,
124+
// SkipPodLevelResources is set to false when PodLevelResources feature is enabled.
125+
SkipPodLevelResources: !podLevelResourcesEnabled,
126+
UseStatusResources: false,
125127
})
126128
// track if limits were applied for each resource.
127129
memoryLimitsDeclared := true
128130
cpuLimitsDeclared := true
129131

130132
limits := resource.PodLimits(allocatedPod, resource.PodResourcesOptions{
133+
// SkipPodLevelResources is set to false when PodLevelResources feature is enabled.
134+
SkipPodLevelResources: !podLevelResourcesEnabled,
131135
ContainerFn: func(res v1.ResourceList, containerType resource.ContainerType) {
132136
if res.Cpu().IsZero() {
133137
cpuLimitsDeclared = false
@@ -137,6 +141,16 @@ func ResourceConfigForPod(allocatedPod *v1.Pod, enforceCPULimits bool, cpuPeriod
137141
}
138142
},
139143
})
144+
145+
if podLevelResourcesEnabled && resource.IsPodLevelResourcesSet(allocatedPod) {
146+
if !allocatedPod.Spec.Resources.Limits.Cpu().IsZero() {
147+
cpuLimitsDeclared = true
148+
}
149+
150+
if !allocatedPod.Spec.Resources.Limits.Memory().IsZero() {
151+
memoryLimitsDeclared = true
152+
}
153+
}
140154
// map hugepage pagesize (bytes) to limits (bytes)
141155
hugePageLimits := HugePageLimits(reqs)
142156

‎pkg/kubelet/cm/helpers_linux_test.go‎

Lines changed: 122 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,11 @@ func TestResourceConfigForPod(t *testing.T) {
7070
cpuNoLimit := int64(-1)
7171
guaranteedMemory := memoryQuantity.Value()
7272
testCases := map[string]struct {
73-
pod *v1.Pod
74-
expected *ResourceConfig
75-
enforceCPULimits bool
76-
quotaPeriod uint64 // in microseconds
73+
pod *v1.Pod
74+
expected *ResourceConfig
75+
enforceCPULimits bool
76+
quotaPeriod uint64 // in microseconds
77+
podLevelResourcesEnabled bool
7778
}{
7879
"besteffort": {
7980
pod: &v1.Pod{
@@ -274,20 +275,134 @@ func TestResourceConfigForPod(t *testing.T) {
274275
quotaPeriod: tunedQuotaPeriod,
275276
expected: &ResourceConfig{CPUShares: &burstablePartialShares},
276277
},
278+
"burstable-with-pod-level-requests": {
279+
pod: &v1.Pod{
280+
Spec: v1.PodSpec{
281+
Resources: &v1.ResourceRequirements{
282+
Requests: getResourceList("100m", "100Mi"),
283+
},
284+
Containers: []v1.Container{
285+
{
286+
Name: "Container with no resources",
287+
},
288+
},
289+
},
290+
},
291+
podLevelResourcesEnabled: true,
292+
enforceCPULimits: true,
293+
quotaPeriod: defaultQuotaPeriod,
294+
expected: &ResourceConfig{CPUShares: &burstableShares},
295+
},
296+
"burstable-with-pod-and-container-level-requests": {
297+
pod: &v1.Pod{
298+
Spec: v1.PodSpec{
299+
Resources: &v1.ResourceRequirements{
300+
Requests: getResourceList("100m", "100Mi"),
301+
},
302+
Containers: []v1.Container{
303+
{
304+
Name: "Container with resources",
305+
Resources: getResourceRequirements(getResourceList("10m", "50Mi"), getResourceList("", "")),
306+
},
307+
},
308+
},
309+
},
310+
podLevelResourcesEnabled: true,
311+
enforceCPULimits: true,
312+
quotaPeriod: defaultQuotaPeriod,
313+
expected: &ResourceConfig{CPUShares: &burstableShares},
314+
},
315+
"burstable-with-pod-level-resources": {
316+
pod: &v1.Pod{
317+
Spec: v1.PodSpec{
318+
Resources: &v1.ResourceRequirements{
319+
Requests: getResourceList("100m", "100Mi"),
320+
Limits: getResourceList("200m", "200Mi"),
321+
},
322+
Containers: []v1.Container{
323+
{
324+
Name: "Container with no resources",
325+
},
326+
},
327+
},
328+
},
329+
podLevelResourcesEnabled: true,
330+
enforceCPULimits: true,
331+
quotaPeriod: defaultQuotaPeriod,
332+
expected: &ResourceConfig{CPUShares: &burstableShares, CPUQuota: &burstableQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &burstableMemory},
333+
},
334+
"burstable-with-pod-and-container-level-resources": {
335+
pod: &v1.Pod{
336+
Spec: v1.PodSpec{
337+
Resources: &v1.ResourceRequirements{
338+
Requests: getResourceList("100m", "100Mi"),
339+
Limits: getResourceList("200m", "200Mi"),
340+
},
341+
Containers: []v1.Container{
342+
{
343+
Name: "Container with resources",
344+
Resources: getResourceRequirements(getResourceList("10m", "50Mi"), getResourceList("50m", "100Mi")),
345+
},
346+
},
347+
},
348+
},
349+
podLevelResourcesEnabled: true,
350+
enforceCPULimits: true,
351+
quotaPeriod: defaultQuotaPeriod,
352+
expected: &ResourceConfig{CPUShares: &burstableShares, CPUQuota: &burstableQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &burstableMemory},
353+
},
354+
"guaranteed-with-pod-level-resources": {
355+
pod: &v1.Pod{
356+
Spec: v1.PodSpec{
357+
Resources: &v1.ResourceRequirements{
358+
Requests: getResourceList("100m", "100Mi"),
359+
Limits: getResourceList("100m", "100Mi"),
360+
},
361+
Containers: []v1.Container{
362+
{
363+
Name: "Container with no resources",
364+
},
365+
},
366+
},
367+
},
368+
podLevelResourcesEnabled: true,
369+
enforceCPULimits: true,
370+
quotaPeriod: defaultQuotaPeriod,
371+
expected: &ResourceConfig{CPUShares: &guaranteedShares, CPUQuota: &guaranteedQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
372+
},
373+
"guaranteed-with-pod-and-container-level-resources": {
374+
pod: &v1.Pod{
375+
Spec: v1.PodSpec{
376+
Resources: &v1.ResourceRequirements{
377+
Requests: getResourceList("100m", "100Mi"),
378+
Limits: getResourceList("100m", "100Mi"),
379+
},
380+
Containers: []v1.Container{
381+
{
382+
Name: "Container with resources",
383+
Resources: getResourceRequirements(getResourceList("10m", "50Mi"), getResourceList("50m", "100Mi")),
384+
},
385+
},
386+
},
387+
},
388+
podLevelResourcesEnabled: true,
389+
enforceCPULimits: true,
390+
quotaPeriod: defaultQuotaPeriod,
391+
expected: &ResourceConfig{CPUShares: &guaranteedShares, CPUQuota: &guaranteedQuota, CPUPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
392+
},
277393
}
278394

279395
for testName, testCase := range testCases {
280-
396+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.PodLevelResources, testCase.podLevelResourcesEnabled)
281397
actual := ResourceConfigForPod(testCase.pod, testCase.enforceCPULimits, testCase.quotaPeriod, false)
282-
283398
if !reflect.DeepEqual(actual.CPUPeriod, testCase.expected.CPUPeriod) {
284399
t.Errorf("unexpected result, test: %v, cpu period not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUPeriod, *actual.CPUPeriod)
285400
}
286401
if !reflect.DeepEqual(actual.CPUQuota, testCase.expected.CPUQuota) {
287402
t.Errorf("unexpected result, test: %v, cpu quota not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUQuota, *actual.CPUQuota)
288403
}
289404
if !reflect.DeepEqual(actual.CPUShares, testCase.expected.CPUShares) {
290-
t.Errorf("unexpected result, test: %v, cpu shares not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUShares, &actual.CPUShares)
405+
t.Errorf("unexpected result, test: %v, cpu shares not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.CPUShares, *actual.CPUShares)
291406
}
292407
if !reflect.DeepEqual(actual.Memory, testCase.expected.Memory) {
293408
t.Errorf("unexpected result, test: %v, memory not as expected. Expected: %v, Actual:%v", testName, *testCase.expected.Memory, *actual.Memory)

‎pkg/kubelet/cm/qos_container_manager_linux.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,11 @@ func (m *qosContainerManagerImpl) setCPUCgroupConfig(configs map[v1.PodQOSClass]
179179
// we only care about the burstable qos tier
180180
continue
181181
}
182-
req := resource.PodRequests(pod, resource.PodResourcesOptions{Reuse: reuseReqs})
182+
req := resource.PodRequests(pod, resource.PodResourcesOptions{
183+
Reuse: reuseReqs,
184+
// SkipPodLevelResources is set to false when PodLevelResources feature is enabled.
185+
SkipPodLevelResources: !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources),
186+
})
183187
if request, found := req[v1.ResourceCPU]; found {
184188
burstablePodCPURequest += request.MilliValue()
185189
}

‎pkg/kubelet/kuberuntime/kuberuntime_container_linux.go‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
v1 "k8s.io/api/core/v1"
3636
"k8s.io/apimachinery/pkg/api/resource"
3737
utilfeature "k8s.io/apiserver/pkg/util/feature"
38+
resourcehelper "k8s.io/component-helpers/resource"
3839
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
3940
"k8s.io/klog/v2"
4041

@@ -95,14 +96,45 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.C
9596
return lc, nil
9697
}
9798

99+
// getCPULimit returns the memory limit for the container to be used to calculate
100+
// Linux Container Resources.
101+
func getCPULimit(pod *v1.Pod, container *v1.Container) *resource.Quantity {
102+
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
103+
// When container-level CPU limit is not set, the pod-level
104+
// limit is used in the calculation for components relying on linux resource limits
105+
// to be set.
106+
if container.Resources.Limits.Cpu().IsZero() {
107+
return pod.Spec.Resources.Limits.Cpu()
108+
}
109+
}
110+
return container.Resources.Limits.Cpu()
111+
}
112+
113+
// getMemoryLimit returns the memory limit for the container to be used to calculate
114+
// Linux Container Resources.
115+
func getMemoryLimit(pod *v1.Pod, container *v1.Container) *resource.Quantity {
116+
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
117+
// When container-level memory limit is not set, the pod-level
118+
// limit is used in the calculation for components relying on linux resource limits
119+
// to be set.
120+
if container.Resources.Limits.Memory().IsZero() {
121+
return pod.Spec.Resources.Limits.Memory()
122+
}
123+
}
124+
return container.Resources.Limits.Memory()
125+
}
126+
98127
// generateLinuxContainerResources generates linux container resources config for runtime
99128
func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(pod *v1.Pod, container *v1.Container, enforceMemoryQoS bool) *runtimeapi.LinuxContainerResources {
100129
// set linux container resources
101130
var cpuRequest *resource.Quantity
102131
if _, cpuRequestExists := container.Resources.Requests[v1.ResourceCPU]; cpuRequestExists {
103132
cpuRequest = container.Resources.Requests.Cpu()
104133
}
105-
lcr := m.calculateLinuxResources(cpuRequest, container.Resources.Limits.Cpu(), container.Resources.Limits.Memory())
134+
135+
memoryLimit := getMemoryLimit(pod, container)
136+
cpuLimit := getCPULimit(pod, container)
137+
lcr := m.calculateLinuxResources(cpuRequest, cpuLimit, memoryLimit)
106138

107139
lcr.OomScoreAdj = int64(qos.GetContainerOOMScoreAdjust(pod, container,
108140
int64(m.machineInfo.MemoryCapacity)))

‎pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -167,13 +167,14 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) {
167167
assert.NoError(t, err)
168168

169169
tests := []struct {
170-
name string
171-
podResources v1.ResourceRequirements
172-
expected *runtimeapi.LinuxContainerResources
170+
name string
171+
containerResources v1.ResourceRequirements
172+
podResources *v1.ResourceRequirements
173+
expected *runtimeapi.LinuxContainerResources
173174
}{
174175
{
175176
name: "Request 128M/1C, Limit 256M/3C",
176-
podResources: v1.ResourceRequirements{
177+
containerResources: v1.ResourceRequirements{
177178
Requests: v1.ResourceList{
178179
v1.ResourceMemory: resource.MustParse("128Mi"),
179180
v1.ResourceCPU: resource.MustParse("1"),
@@ -192,7 +193,7 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) {
192193
},
193194
{
194195
name: "Request 128M/2C, No Limit",
195-
podResources: v1.ResourceRequirements{
196+
containerResources: v1.ResourceRequirements{
196197
Requests: v1.ResourceList{
197198
v1.ResourceMemory: resource.MustParse("128Mi"),
198199
v1.ResourceCPU: resource.MustParse("2"),
@@ -205,6 +206,27 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) {
205206
MemoryLimitInBytes: 0,
206207
},
207208
},
209+
{
210+
name: "Container Level Request 128M/1C, Pod Level Limit 256M/3C",
211+
containerResources: v1.ResourceRequirements{
212+
Requests: v1.ResourceList{
213+
v1.ResourceMemory: resource.MustParse("128Mi"),
214+
v1.ResourceCPU: resource.MustParse("1"),
215+
},
216+
},
217+
podResources: &v1.ResourceRequirements{
218+
Limits: v1.ResourceList{
219+
v1.ResourceMemory: resource.MustParse("256Mi"),
220+
v1.ResourceCPU: resource.MustParse("3"),
221+
},
222+
},
223+
expected: &runtimeapi.LinuxContainerResources{
224+
CpuPeriod: 100000,
225+
CpuQuota: 300000,
226+
CpuShares: 1024,
227+
MemoryLimitInBytes: 256 * 1024 * 1024,
228+
},
229+
},
208230
}
209231

210232
for _, test := range tests {
@@ -222,12 +244,17 @@ func TestGenerateLinuxContainerConfigResources(t *testing.T) {
222244
ImagePullPolicy: v1.PullIfNotPresent,
223245
Command: []string{"testCommand"},
224246
WorkingDir: "testWorkingDir",
225-
Resources: test.podResources,
247+
Resources: test.containerResources,
226248
},
227249
},
228250
},
229251
}
230252

253+
if test.podResources != nil {
254+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, true)
255+
pod.Spec.Resources = test.podResources
256+
}
257+
231258
linuxConfig, err := m.generateLinuxContainerConfig(&pod.Spec.Containers[0], pod, new(int64), "", nil, false)
232259
assert.NoError(t, err)
233260
assert.Equal(t, test.expected.CpuPeriod, linuxConfig.GetResources().CpuPeriod, test.name)

‎pkg/kubelet/kuberuntime/kuberuntime_sandbox_linux.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ package kuberuntime
2222
import (
2323
v1 "k8s.io/api/core/v1"
2424
"k8s.io/apimachinery/pkg/api/resource"
25+
utilfeature "k8s.io/apiserver/pkg/util/feature"
2526
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
27+
"k8s.io/kubernetes/pkg/features"
2628

2729
resourcehelper "k8s.io/component-helpers/resource"
2830
)
@@ -44,6 +46,8 @@ func (m *kubeGenericRuntimeManager) convertOverheadToLinuxResources(pod *v1.Pod)
4446
func (m *kubeGenericRuntimeManager) calculateSandboxResources(pod *v1.Pod) *runtimeapi.LinuxContainerResources {
4547
opts := resourcehelper.PodResourcesOptions{
4648
ExcludeOverhead: true,
49+
// SkipPodLevelResources is set to false when PodLevelResources feature is enabled.
50+
SkipPodLevelResources: !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources),
4751
}
4852
req := resourcehelper.PodRequests(pod, opts)
4953
lim := resourcehelper.PodLimits(pod, opts)

0 commit comments

Comments
 (0)