Skip to content

Commit 568255a

Browse files
haircommanderbitoku
haircommander
authored and
bitoku
committed
e2e_node: add test about container metrics from cadvisor
Signed-off-by: Peter Hunt <[email protected]> Signed-off-by: Ayato Tokubi <[email protected]>
1 parent 5fc7032 commit 568255a

File tree

2 files changed

+182
-30
lines changed

2 files changed

+182
-30
lines changed

‎test/e2e_node/container_metrics_test.go‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2enode
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"time"
23+
24+
"github.com/onsi/ginkgo/v2"
25+
"github.com/onsi/gomega"
26+
"github.com/onsi/gomega/gstruct"
27+
"github.com/onsi/gomega/types"
28+
29+
"k8s.io/kubernetes/test/e2e/framework"
30+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
31+
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
32+
admissionapi "k8s.io/pod-security-admission/api"
33+
)
34+
35+
var _ = SIGDescribe("ContainerMetrics", "[LinuxOnly]", framework.WithNodeConformance(), func() {
36+
f := framework.NewDefaultFramework("container-metrics")
37+
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
38+
ginkgo.Context("when querying /metrics/cadvisor", func() {
39+
ginkgo.BeforeEach(func(ctx context.Context) {
40+
createMetricsPods(ctx, f)
41+
})
42+
ginkgo.AfterEach(func(ctx context.Context) {
43+
removeMetricsPods(ctx, f)
44+
})
45+
ginkgo.It("should report container metrics", func(ctx context.Context) {
46+
keys := gstruct.Keys{}
47+
ctrMatches := map[string]types.GomegaMatcher{
48+
"container_blkio_device_usage_total": boundedSample(0, 10000000),
49+
"container_cpu_load_average_10s": boundedSample(0, 100),
50+
"container_cpu_system_seconds_total": boundedSample(0, 100),
51+
"container_cpu_usage_seconds_total": boundedSample(0, 100),
52+
"container_cpu_user_seconds_total": boundedSample(0, 100),
53+
"container_file_descriptors": boundedSample(0, 100),
54+
"container_fs_reads_bytes_total": boundedSample(0, 10000000),
55+
"container_fs_reads_total": boundedSample(0, 100),
56+
"container_fs_usage_bytes": boundedSample(0, 1000000),
57+
"container_fs_writes_bytes_total": boundedSample(0, 1000000),
58+
"container_fs_writes_total": boundedSample(0, 100),
59+
"container_last_seen": boundedSample(time.Now().Add(-maxStatsAge).Unix(), time.Now().Add(2*time.Minute).Unix()),
60+
"container_memory_cache": boundedSample(1*e2evolume.Kb, 10*e2evolume.Mb),
61+
"container_memory_failcnt": preciseSample(0),
62+
"container_memory_failures_total": boundedSample(0, 1000000),
63+
"container_memory_mapped_file": boundedSample(0, 10000000),
64+
"container_memory_max_usage_bytes": boundedSample(0, 80*e2evolume.Mb),
65+
"container_memory_rss": boundedSample(10*e2evolume.Kb, 80*e2evolume.Mb),
66+
"container_memory_swap": preciseSample(0),
67+
"container_memory_usage_bytes": boundedSample(10*e2evolume.Kb, 80*e2evolume.Mb),
68+
"container_memory_working_set_bytes": boundedSample(10*e2evolume.Kb, 80*e2evolume.Mb),
69+
"container_oom_events_total": preciseSample(0),
70+
"container_processes": boundedSample(0, 10),
71+
"container_sockets": boundedSample(0, 10),
72+
"container_spec_cpu_period": preciseSample(100000),
73+
"container_spec_cpu_shares": preciseSample(2),
74+
"container_spec_memory_limit_bytes": preciseSample(79998976),
75+
"container_spec_memory_reservation_limit_bytes": preciseSample(0),
76+
"container_spec_memory_swap_limit_bytes": preciseSample(0),
77+
"container_start_time_seconds": boundedSample(time.Now().Add(-maxStatsAge).Unix(), time.Now().Add(2*time.Minute).Unix()),
78+
"container_tasks_state": preciseSample(0),
79+
"container_threads": boundedSample(0, 10),
80+
"container_threads_max": boundedSample(0, 100000),
81+
"container_ulimits_soft": boundedSample(0, 10000000),
82+
}
83+
appendMatchesForContainer(f.Namespace.Name, pod0, pod1, "busybox-container", ctrMatches, keys, gstruct.AllowDuplicates|gstruct.IgnoreExtras)
84+
85+
ctrOptionalMatches := map[string]types.GomegaMatcher{
86+
"container_fs_io_current": boundedSample(0, 100),
87+
"container_fs_io_time_seconds_total": boundedSample(0, 100),
88+
"container_fs_io_time_weighted_seconds_total": boundedSample(0, 100),
89+
"container_fs_inodes_free": boundedSample(0, 10*e2evolume.Kb),
90+
"container_fs_inodes_total": boundedSample(0, 100),
91+
"container_fs_limit_bytes": boundedSample(100*e2evolume.Mb, 10*e2evolume.Tb),
92+
"container_fs_usage_bytes": boundedSample(0, 1000000),
93+
"container_fs_read_seconds_total": preciseSample(0),
94+
"container_fs_reads_merged_total": preciseSample(0),
95+
"container_fs_sector_reads_total": preciseSample(0),
96+
"container_fs_sector_writes_total": preciseSample(0),
97+
"container_fs_write_seconds_total": preciseSample(0),
98+
"container_fs_writes_merged_total": preciseSample(0),
99+
}
100+
// Missing from containerd, so set gstruct.IgnoreMissing
101+
// See https://github.com/google/cadvisor/issues/2785
102+
appendMatchesForContainer(f.Namespace.Name, pod0, pod1, "busybox-container", ctrOptionalMatches, keys, gstruct.AllowDuplicates|gstruct.IgnoreMissing|gstruct.IgnoreExtras)
103+
104+
podMatches := map[string]types.GomegaMatcher{
105+
"container_network_receive_bytes_total": boundedSample(10, 10*e2evolume.Mb),
106+
"container_network_receive_errors_total": boundedSample(0, 1000),
107+
"container_network_receive_packets_dropped_total": boundedSample(0, 1000),
108+
"container_network_receive_packets_total": boundedSample(0, 1000),
109+
"container_network_transmit_bytes_total": boundedSample(10, 10*e2evolume.Mb),
110+
"container_network_transmit_errors_total": boundedSample(0, 1000),
111+
"container_network_transmit_packets_dropped_total": boundedSample(0, 1000),
112+
"container_network_transmit_packets_total": boundedSample(0, 1000),
113+
}
114+
// TODO: determine why these are missing from containerd but not CRI-O
115+
appendMatchesForContainer(f.Namespace.Name, pod0, pod1, "POD", podMatches, keys, gstruct.AllowDuplicates|gstruct.IgnoreMissing|gstruct.IgnoreExtras)
116+
117+
matchResourceMetrics := gstruct.MatchKeys(gstruct.IgnoreExtras, keys)
118+
ginkgo.By("Giving pods a minute to start up and produce metrics")
119+
gomega.Eventually(ctx, getContainerMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
120+
ginkgo.By("Ensuring the metrics match the expectations a few more times")
121+
gomega.Consistently(ctx, getContainerMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
122+
})
123+
})
124+
})
125+
126+
func getContainerMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) {
127+
ginkgo.By("getting container metrics from cadvisor")
128+
return e2emetrics.GrabKubeletMetricsWithoutProxy(ctx, framework.TestContext.NodeName+":10255", "/metrics/cadvisor")
129+
}
130+
131+
func preciseSample(value interface{}) types.GomegaMatcher {
132+
return gstruct.PointTo(gstruct.MatchAllFields(gstruct.Fields{
133+
"Metric": gstruct.Ignore(),
134+
"Value": gomega.BeEquivalentTo(value),
135+
"Timestamp": gstruct.Ignore(),
136+
"Histogram": gstruct.Ignore(),
137+
}))
138+
}
139+
140+
func appendMatchesForContainer(ns, pod1, pod2, ctr string, matches map[string]types.GomegaMatcher, keys gstruct.Keys, options gstruct.Options) {
141+
for k, v := range matches {
142+
keys[k] = gstruct.MatchElements(containerID, options, gstruct.Elements{
143+
fmt.Sprintf("%s::%s::%s", ns, pod1, ctr): v,
144+
fmt.Sprintf("%s::%s::%s", ns, pod2, ctr): v,
145+
})
146+
}
147+
}

‎test/e2e_node/resource_metrics_test.go‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -49,24 +49,7 @@ var _ = SIGDescribe("ResourceMetricsAPI", nodefeature.ResourceMetrics, func() {
4949
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
5050
ginkgo.Context("when querying /resource/metrics", func() {
5151
ginkgo.BeforeEach(func(ctx context.Context) {
52-
ginkgo.By("Creating test pods to measure their resource usage")
53-
numRestarts := int32(1)
54-
pods := getSummaryTestPods(f, numRestarts, pod0, pod1)
55-
e2epod.NewPodClient(f).CreateBatch(ctx, pods)
56-
57-
ginkgo.By("restarting the containers to ensure container metrics are still being gathered after a container is restarted")
58-
gomega.Eventually(ctx, func(ctx context.Context) error {
59-
for _, pod := range pods {
60-
err := verifyPodRestartCount(ctx, f, pod.Name, len(pod.Spec.Containers), numRestarts)
61-
if err != nil {
62-
return err
63-
}
64-
}
65-
return nil
66-
}, time.Minute, 5*time.Second).Should(gomega.Succeed())
67-
68-
ginkgo.By("Waiting 15 seconds for cAdvisor to collect 2 stats points")
69-
time.Sleep(15 * time.Second)
52+
createMetricsPods(ctx, f)
7053
})
7154
ginkgo.It("should report resource usage through the resource metrics api", func(ctx context.Context) {
7255
ginkgo.By("Fetching node so we can match against an appropriate memory limit")
@@ -122,22 +105,29 @@ var _ = SIGDescribe("ResourceMetricsAPI", nodefeature.ResourceMetrics, func() {
122105
gomega.Consistently(ctx, getResourceMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
123106
})
124107
ginkgo.AfterEach(func(ctx context.Context) {
125-
ginkgo.By("Deleting test pods")
126-
var zero int64 = 0
127-
e2epod.NewPodClient(f).DeleteSync(ctx, pod0, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
128-
e2epod.NewPodClient(f).DeleteSync(ctx, pod1, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
129-
if !ginkgo.CurrentSpecReport().Failed() {
130-
return
131-
}
132-
if framework.TestContext.DumpLogsOnFailure {
133-
e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf)
134-
}
135-
ginkgo.By("Recording processes in system cgroups")
136-
recordSystemCgroupProcesses(ctx)
108+
removeMetricsPods(ctx, f)
137109
})
138110
})
139111
})
140112

113+
func createMetricsPods(ctx context.Context, f *framework.Framework) {
114+
ginkgo.By("Creating test pods to measure their resource usage")
115+
numRestarts := int32(1)
116+
pods := getSummaryTestPods(f, numRestarts, pod0, pod1)
117+
e2epod.NewPodClient(f).CreateBatch(ctx, pods)
118+
119+
ginkgo.By("Restarting the containers to ensure container metrics are still being gathered after a container is restarted")
120+
gomega.Eventually(ctx, func(ctx context.Context) error {
121+
for _, pod := range pods {
122+
err := verifyPodRestartCount(ctx, f, pod.Name, len(pod.Spec.Containers), numRestarts)
123+
if err != nil {
124+
return err
125+
}
126+
}
127+
return nil
128+
}, time.Minute, 5*time.Second).Should(gomega.Succeed())
129+
}
130+
141131
func getResourceMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) {
142132
ginkgo.By("getting stable resource metrics API")
143133
return e2emetrics.GrabKubeletMetricsWithoutProxy(ctx, nodeNameOrIP()+":10255", "/metrics/resource")
@@ -193,3 +183,18 @@ func haveKeys(keys ...string) types.GomegaMatcher {
193183

194184
return matcher
195185
}
186+
187+
func removeMetricsPods(ctx context.Context, f *framework.Framework) {
188+
ginkgo.By("Deleting test pods")
189+
var zero int64 = 0
190+
e2epod.NewPodClient(f).DeleteSync(ctx, pod0, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
191+
e2epod.NewPodClient(f).DeleteSync(ctx, pod1, metav1.DeleteOptions{GracePeriodSeconds: &zero}, 10*time.Minute)
192+
if !ginkgo.CurrentSpecReport().Failed() {
193+
return
194+
}
195+
if framework.TestContext.DumpLogsOnFailure {
196+
e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf)
197+
}
198+
ginkgo.By("Recording processes in system cgroups")
199+
recordSystemCgroupProcesses(ctx)
200+
}

0 commit comments

Comments
 (0)