Skip to content

Commit 3be3153

Browse files
ashleycutaloashleycutalo
committed
cri_stats_provider: do not consider exited containers when calculating cpu usage
1 parent 60044a8 commit 3be3153

File tree

2 files changed

+29
-32
lines changed

2 files changed

+29
-32
lines changed

‎pkg/kubelet/stats/cri_stats_provider.go‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -733,9 +733,8 @@ func removeTerminatedPods(pods []*runtimeapi.PodSandbox) []*runtimeapi.PodSandbo
733733
return result
734734
}
735735

736-
// removeTerminatedContainers returns containers with terminated ones.
737-
// It only removes a terminated container when there is a running instance
738-
// of the container.
736+
// removeTerminatedContainers removes all terminated containers since they should
737+
// not be used for usage calculations.
739738
func removeTerminatedContainers(containers []*runtimeapi.Container) []*runtimeapi.Container {
740739
containerMap := make(map[containerID][]*runtimeapi.Container)
741740
// Sort order by create time
@@ -752,20 +751,11 @@ func removeTerminatedContainers(containers []*runtimeapi.Container) []*runtimeap
752751

753752
result := make([]*runtimeapi.Container, 0)
754753
for _, refs := range containerMap {
755-
if len(refs) == 1 {
756-
result = append(result, refs[0])
757-
continue
758-
}
759-
found := false
760754
for i := 0; i < len(refs); i++ {
761755
if refs[i].State == runtimeapi.ContainerState_CONTAINER_RUNNING {
762-
found = true
763756
result = append(result, refs[i])
764757
}
765758
}
766-
if !found {
767-
result = append(result, refs[len(refs)-1])
768-
}
769759
}
770760
return result
771761
}

‎pkg/kubelet/stats/cri_stats_provider_test.go‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ const (
7777
cName5 = "container5-name"
7878
cName6 = "container6-name"
7979
cName7 = "container7-name"
80+
cName8 = "container8-name"
8081
)
8182

8283
func TestCRIListPodStats(t *testing.T) {
@@ -109,10 +110,14 @@ func TestCRIListPodStats(t *testing.T) {
109110
containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
110111
containerLogStats4 = makeFakeLogStats(4000)
111112

113+
// Running pod with a terminated container and a running container
112114
sandbox3 = makeFakePodSandbox("sandbox3-name", "sandbox3-uid", "sandbox3-ns", false)
115+
sandbox3Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
113116
container5 = makeFakeContainer(sandbox3, cName5, 0, true)
114117
containerStats5 = makeFakeContainerStats(container5, imageFsMountpoint)
115118
containerLogStats5 = makeFakeLogStats(5000)
119+
container8 = makeFakeContainer(sandbox3, cName8, 0, false)
120+
containerStats8 = makeFakeContainerStats(container8, imageFsMountpoint)
116121

117122
// Terminated pod sandbox
118123
sandbox4 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", true)
@@ -153,6 +158,7 @@ func TestCRIListPodStats(t *testing.T) {
153158
sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
154159
sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""),
155160
container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
161+
sandbox3Cgroup: getTestContainerInfo(seedSandbox3, "", "", ""),
156162
}
157163

158164
options := cadvisorapiv2.RequestOptions{
@@ -170,10 +176,10 @@ func TestCRIListPodStats(t *testing.T) {
170176
sandbox0, sandbox1, sandbox2, sandbox3, sandbox4, sandbox5,
171177
})
172178
fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
173-
container0, container1, container2, container3, container4, container5, container6, container7,
179+
container0, container1, container2, container3, container4, container5, container6, container7, container8,
174180
})
175181
fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
176-
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, containerStats6, containerStats7,
182+
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, containerStats6, containerStats7, containerStats8
177183
})
178184

179185
ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
@@ -296,14 +302,12 @@ func TestCRIListPodStats(t *testing.T) {
296302
assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano())
297303
assert.Equal(1, len(p3.Containers))
298304

299-
c5 := p3.Containers[0]
300-
assert.Equal(cName5, c5.Name)
301-
assert.Equal(container5.CreatedAt, c5.StartTime.UnixNano())
302-
assert.NotNil(c5.CPU.Time)
303-
assert.Zero(*c5.CPU.UsageCoreNanoSeconds)
304-
assert.Zero(*c5.CPU.UsageNanoCores)
305-
assert.NotNil(c5.Memory.Time)
306-
assert.Zero(*c5.Memory.WorkingSetBytes)
305+
c8 := p3.Containers[0]
306+
assert.Equal(cName8, c8.Name)
307+
assert.Equal(container8.CreatedAt, c8.StartTime.UnixNano())
308+
assert.NotNil(c8.CPU.Time)
309+
assert.NotNil(c8.Memory.Time)
310+
checkCRIPodCPUAndMemoryStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
307311

308312
mockCadvisor.AssertExpectations(t)
309313
}
@@ -333,9 +337,13 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
333337
container4 = makeFakeContainer(sandbox2, cName3, 1, false)
334338
containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
335339

340+
// Running pod with a terminated container and a running container
336341
sandbox3 = makeFakePodSandbox("sandbox3-name", "sandbox3-uid", "sandbox3-ns", false)
342+
sandbox3Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
337343
container5 = makeFakeContainer(sandbox3, cName5, 0, true)
338344
containerStats5 = makeFakeContainerStats(container5, imageFsMountpoint)
345+
container8 = makeFakeContainer(sandbox3, cName8, 0, false)
346+
containerStats8 = makeFakeContainerStats(container8, imageFsMountpoint)
339347

340348
// Terminated pod sandbox
341349
sandbox4 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", true)
@@ -370,6 +378,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
370378
sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
371379
sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""),
372380
container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
381+
sandbox3Cgroup: getTestContainerInfo(seedSandbox3, "", "", ""),
373382
}
374383

375384
options := cadvisorapiv2.RequestOptions{
@@ -384,10 +393,10 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
384393
sandbox0, sandbox1, sandbox2, sandbox3, sandbox4, sandbox5,
385394
})
386395
fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
387-
container0, container1, container2, container3, container4, container5, container6, container7,
396+
container0, container1, container2, container3, container4, container5, container6, container7, container8,
388397
})
389398
fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
390-
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, containerStats6, containerStats7,
399+
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, containerStats6, containerStats7, containerStats8,
391400
})
392401

393402
ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
@@ -484,14 +493,12 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
484493
assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano())
485494
assert.Equal(1, len(p3.Containers))
486495

487-
c5 := p3.Containers[0]
488-
assert.Equal(cName5, c5.Name)
489-
assert.Equal(container5.CreatedAt, c5.StartTime.UnixNano())
490-
assert.NotNil(c5.CPU.Time)
491-
assert.Zero(*c5.CPU.UsageCoreNanoSeconds)
492-
assert.Zero(*c5.CPU.UsageNanoCores)
493-
assert.NotNil(c5.Memory.Time)
494-
assert.Zero(*c5.Memory.WorkingSetBytes)
496+
c8 := p3.Containers[0]
497+
assert.Equal(cName8, c8.Name)
498+
assert.Equal(container8.CreatedAt, c8.StartTime.UnixNano())
499+
assert.NotNil(c8.CPU.Time)
500+
assert.NotNil(c8.Memory.Time)
501+
checkCRIPodCPUAndMemoryStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
495502

496503
mockCadvisor.AssertExpectations(t)
497504
}

0 commit comments

Comments
 (0)