Skip to content

Commit 348fd08

Browse files
committed
Refactored metrics-related functions from framework/metrics_util.go
This a refactoring of framework/metrics_utils.go into framework/metrics. Signed-off-by: alejandrox1 <[email protected]>
1 parent e79dcc2 commit 348fd08

29 files changed

+1318
-988
lines changed

hack/.golint_failures

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,7 @@ staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1
558558
staging/src/k8s.io/sample-apiserver/pkg/registry/wardle/fischer
559559
staging/src/k8s.io/sample-apiserver/pkg/registry/wardle/flunder
560560
test/e2e/common
561+
test/e2e/framework/metrics
561562
test/e2e/lifecycle/bootstrap
562563
test/e2e/storage/vsphere
563564
test/e2e_kubeadm

test/e2e/apimachinery/garbage_collector.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import (
3939
clientset "k8s.io/client-go/kubernetes"
4040
"k8s.io/kubernetes/test/e2e/framework"
4141
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
42-
"k8s.io/kubernetes/test/e2e/framework/metrics"
42+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
4343

4444
"github.com/onsi/ginkgo"
4545
imageutils "k8s.io/kubernetes/test/utils/image"
@@ -242,15 +242,15 @@ func verifyRemainingObjects(f *framework.Framework, objects map[string]int) (boo
242242
func gatherMetrics(f *framework.Framework) {
243243
ginkgo.By("Gathering metrics")
244244
var summary framework.TestDataSummary
245-
grabber, err := metrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, false, false, true, false, false)
245+
grabber, err := e2emetrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, false, false, true, false, false)
246246
if err != nil {
247247
e2elog.Logf("Failed to create MetricsGrabber. Skipping metrics gathering.")
248248
} else {
249249
received, err := grabber.Grab()
250250
if err != nil {
251251
e2elog.Logf("MetricsGrabber failed grab metrics. Skipping metrics gathering.")
252252
} else {
253-
summary = (*framework.MetricsForE2E)(&received)
253+
summary = (*e2emetrics.MetricsForE2E)(&received)
254254
e2elog.Logf(summary.PrintHumanReadable())
255255
}
256256
}

test/e2e/e2e.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import (
3939
"k8s.io/kubernetes/test/e2e/framework"
4040
"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
4141
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
42-
"k8s.io/kubernetes/test/e2e/framework/metrics"
42+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
4343
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
4444
"k8s.io/kubernetes/test/e2e/manifest"
4545
testutils "k8s.io/kubernetes/test/utils"
@@ -188,7 +188,7 @@ func gatherTestSuiteMetrics() error {
188188
}
189189

190190
// Grab metrics for apiserver, scheduler, controller-manager, kubelet (for non-kubemark case) and cluster autoscaler (optionally).
191-
grabber, err := metrics.NewMetricsGrabber(c, nil, !framework.ProviderIs("kubemark"), true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics)
191+
grabber, err := e2emetrics.NewMetricsGrabber(c, nil, !framework.ProviderIs("kubemark"), true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics)
192192
if err != nil {
193193
return fmt.Errorf("failed to create MetricsGrabber: %v", err)
194194
}
@@ -198,7 +198,7 @@ func gatherTestSuiteMetrics() error {
198198
return fmt.Errorf("failed to grab metrics: %v", err)
199199
}
200200

201-
metricsForE2E := (*framework.MetricsForE2E)(&received)
201+
metricsForE2E := (*e2emetrics.MetricsForE2E)(&received)
202202
metricsJSON := metricsForE2E.PrintJSON()
203203
if framework.TestContext.ReportDir != "" {
204204
filePath := path.Join(framework.TestContext.ReportDir, "MetricsForE2ESuite_"+time.Now().Format(time.RFC3339)+".json")

test/e2e/framework/BUILD

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ go_library(
1414
"google_compute.go",
1515
"kubelet_stats.go",
1616
"log_size_monitoring.go",
17-
"metrics_util.go",
1817
"networking_utils.go",
1918
"nodes_util.go",
2019
"perf_util.go",
@@ -48,7 +47,6 @@ go_library(
4847
"//pkg/master/ports:go_default_library",
4948
"//pkg/registry/core/service/portallocator:go_default_library",
5049
"//pkg/scheduler/algorithm/predicates:go_default_library",
51-
"//pkg/scheduler/metrics:go_default_library",
5250
"//pkg/scheduler/nodeinfo:go_default_library",
5351
"//pkg/security/podsecuritypolicy/seccomp:go_default_library",
5452
"//pkg/util/system:go_default_library",
@@ -113,7 +111,6 @@ go_library(
113111
"//vendor/github.com/onsi/gomega:go_default_library",
114112
"//vendor/github.com/onsi/gomega/types:go_default_library",
115113
"//vendor/github.com/pkg/errors:go_default_library",
116-
"//vendor/github.com/prometheus/common/expfmt:go_default_library",
117114
"//vendor/github.com/prometheus/common/model:go_default_library",
118115
"//vendor/golang.org/x/net/websocket:go_default_library",
119116
"//vendor/k8s.io/klog:go_default_library",

test/e2e/framework/flake_reporting_util.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"sync"
2323

2424
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
25+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
2526
)
2627

2728
// FlakeReport is a struct for managing the flake report.
@@ -90,7 +91,7 @@ func (f *FlakeReport) PrintHumanReadable() string {
9091
func (f *FlakeReport) PrintJSON() string {
9192
f.lock.RLock()
9293
defer f.lock.RUnlock()
93-
return PrettyPrintJSON(f)
94+
return e2emetrics.PrettyPrintJSON(f)
9495
}
9596

9697
// SummaryKind returns the summary of flake report.

test/e2e/framework/framework.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ import (
4747
"k8s.io/client-go/restmapper"
4848
scaleclient "k8s.io/client-go/scale"
4949
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
50-
"k8s.io/kubernetes/test/e2e/framework/metrics"
50+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
5151
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
5252
testutils "k8s.io/kubernetes/test/utils"
5353

@@ -112,7 +112,7 @@ type Framework struct {
112112
TestSummaries []TestDataSummary
113113

114114
// Place to keep ClusterAutoscaler metrics from before test in order to compute delta.
115-
clusterAutoscalerMetricsBeforeTest metrics.Collection
115+
clusterAutoscalerMetricsBeforeTest e2emetrics.Collection
116116
}
117117

118118
// TestDataSummary is an interface for managing test data.
@@ -271,7 +271,7 @@ func (f *Framework) BeforeEach() {
271271

272272
gatherMetricsAfterTest := TestContext.GatherMetricsAfterTest == "true" || TestContext.GatherMetricsAfterTest == "master"
273273
if gatherMetricsAfterTest && TestContext.IncludeClusterAutoscalerMetrics {
274-
grabber, err := metrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, !ProviderIs("kubemark"), false, false, false, TestContext.IncludeClusterAutoscalerMetrics)
274+
grabber, err := e2emetrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, !ProviderIs("kubemark"), false, false, false, TestContext.IncludeClusterAutoscalerMetrics)
275275
if err != nil {
276276
e2elog.Logf("Failed to create MetricsGrabber (skipping ClusterAutoscaler metrics gathering before test): %v", err)
277277
} else {
@@ -363,16 +363,16 @@ func (f *Framework) AfterEach() {
363363
ginkgo.By("Gathering metrics")
364364
// Grab apiserver, scheduler, controller-manager metrics and (optionally) nodes' kubelet metrics.
365365
grabMetricsFromKubelets := TestContext.GatherMetricsAfterTest != "master" && !ProviderIs("kubemark")
366-
grabber, err := metrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, grabMetricsFromKubelets, true, true, true, TestContext.IncludeClusterAutoscalerMetrics)
366+
grabber, err := e2emetrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, grabMetricsFromKubelets, true, true, true, TestContext.IncludeClusterAutoscalerMetrics)
367367
if err != nil {
368368
e2elog.Logf("Failed to create MetricsGrabber (skipping metrics gathering): %v", err)
369369
} else {
370370
received, err := grabber.Grab()
371371
if err != nil {
372372
e2elog.Logf("MetricsGrabber failed to grab some of the metrics: %v", err)
373373
}
374-
(*MetricsForE2E)(&received).computeClusterAutoscalerMetricsDelta(f.clusterAutoscalerMetricsBeforeTest)
375-
f.TestSummaries = append(f.TestSummaries, (*MetricsForE2E)(&received))
374+
(*e2emetrics.MetricsForE2E)(&received).ComputeClusterAutoscalerMetricsDelta(f.clusterAutoscalerMetricsBeforeTest)
375+
f.TestSummaries = append(f.TestSummaries, (*e2emetrics.MetricsForE2E)(&received))
376376
}
377377
}
378378

test/e2e/framework/kubelet_stats.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import (
4747

4848
// KubeletLatencyMetric stores metrics scraped from the kubelet server's /metric endpoint.
4949
// TODO: Get some more structure around the metrics and this type
50+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
5051
type KubeletLatencyMetric struct {
5152
// eg: list, info, create
5253
Operation string
@@ -59,6 +60,7 @@ type KubeletLatencyMetric struct {
5960

6061
// KubeletLatencyMetrics implements sort.Interface for []KubeletMetric based on
6162
// the latency field.
63+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
6264
type KubeletLatencyMetrics []KubeletLatencyMetric
6365

6466
func (a KubeletLatencyMetrics) Len() int { return len(a) }
@@ -67,6 +69,7 @@ func (a KubeletLatencyMetrics) Less(i, j int) bool { return a[i].Latency > a[j].
6769

6870
// If a apiserver client is passed in, the function will try to get kubelet metrics from metrics grabber;
6971
// or else, the function will try to get kubelet metrics directly from the node.
72+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
7073
func getKubeletMetricsFromNode(c clientset.Interface, nodeName string) (metrics.KubeletMetrics, error) {
7174
if c == nil {
7275
return metrics.GrabKubeletMetricsWithoutProxy(nodeName, "/metrics")
@@ -80,6 +83,7 @@ func getKubeletMetricsFromNode(c clientset.Interface, nodeName string) (metrics.
8083

8184
// getKubeletMetrics gets all metrics in kubelet subsystem from specified node and trims
8285
// the subsystem prefix.
86+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
8387
func getKubeletMetrics(c clientset.Interface, nodeName string) (metrics.KubeletMetrics, error) {
8488
ms, err := getKubeletMetricsFromNode(c, nodeName)
8589
if err != nil {
@@ -102,6 +106,7 @@ func getKubeletMetrics(c clientset.Interface, nodeName string) (metrics.KubeletM
102106
// GetDefaultKubeletLatencyMetrics calls GetKubeletLatencyMetrics with a set of default metricNames
103107
// identifying common latency metrics.
104108
// Note that the KubeletMetrics passed in should not contain subsystem prefix.
109+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
105110
func GetDefaultKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
106111
latencyMetricNames := sets.NewString(
107112
kubeletmetrics.PodWorkerDurationKey,
@@ -117,6 +122,7 @@ func GetDefaultKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMe
117122

118123
// GetKubeletLatencyMetrics filters ms to include only those contained in the metricNames set,
119124
// then constructs a KubeletLatencyMetrics list based on the samples associated with those metrics.
125+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
120126
func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics, filterMetricNames sets.String) KubeletLatencyMetrics {
121127
var latencyMetrics KubeletLatencyMetrics
122128
for name, samples := range ms {
@@ -266,6 +272,7 @@ func getNodeRuntimeOperationErrorRate(c clientset.Interface, node string) (NodeR
266272
}
267273

268274
// HighLatencyKubeletOperations logs and counts the high latency metrics exported by the kubelet server via /metrics.
275+
// TODO(alejandrox1): this is already present in test/e2e/framework/metrics.
269276
func HighLatencyKubeletOperations(c clientset.Interface, threshold time.Duration, nodeName string, logFunc func(fmt string, args ...interface{})) (KubeletLatencyMetrics, error) {
270277
ms, err := getKubeletMetrics(c, nodeName)
271278
if err != nil {

test/e2e/framework/log_size_monitoring.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
clientset "k8s.io/client-go/kubernetes"
2929
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
30+
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
3031
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
3132
)
3233

@@ -108,7 +109,7 @@ func (s *LogsSizeDataSummary) PrintHumanReadable() string {
108109

109110
// PrintJSON returns the summary of log size data with JSON format.
110111
func (s *LogsSizeDataSummary) PrintJSON() string {
111-
return PrettyPrintJSON(*s)
112+
return e2emetrics.PrettyPrintJSON(*s)
112113
}
113114

114115
// SummaryKind returns the summary of log size data summary.

test/e2e/framework/metrics/BUILD

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,37 @@ load(
88
go_library(
99
name = "go_default_library",
1010
srcs = [
11+
"api.go",
1112
"api_server_metrics.go",
1213
"cluster_autoscaler_metrics.go",
1314
"controller_manager_metrics.go",
15+
"e2e_metrics.go",
16+
"etcd.go",
1417
"generic_metrics.go",
18+
"interesting_metrics.go",
1519
"kubelet_metrics.go",
20+
"latencies.go",
1621
"metrics_grabber.go",
22+
"pod.go",
1723
"scheduler_metrics.go",
24+
"scheduling.go",
1825
],
1926
importpath = "k8s.io/kubernetes/test/e2e/framework/metrics",
2027
deps = [
2128
"//pkg/apis/core:go_default_library",
29+
"//pkg/kubelet/dockershim/metrics:go_default_library",
30+
"//pkg/kubelet/metrics:go_default_library",
2231
"//pkg/master/ports:go_default_library",
32+
"//pkg/scheduler/metrics:go_default_library",
2333
"//pkg/util/system:go_default_library",
2434
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
2535
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
36+
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
2637
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
38+
"//test/e2e/framework/log:go_default_library",
39+
"//test/e2e/framework/ssh:go_default_library",
40+
"//test/e2e/perftype:go_default_library",
41+
"//vendor/github.com/onsi/gomega:go_default_library",
2742
"//vendor/github.com/prometheus/common/expfmt:go_default_library",
2843
"//vendor/github.com/prometheus/common/model:go_default_library",
2944
"//vendor/k8s.io/klog:go_default_library",

test/e2e/framework/metrics/api.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package metrics
18+
19+
import (
20+
"fmt"
21+
"time"
22+
23+
e2eperftype "k8s.io/kubernetes/test/e2e/perftype"
24+
)
25+
26+
// APICall is a struct for managing API call.
27+
type APICall struct {
28+
Resource string `json:"resource"`
29+
Subresource string `json:"subresource"`
30+
Verb string `json:"verb"`
31+
Scope string `json:"scope"`
32+
Latency LatencyMetric `json:"latency"`
33+
Count int `json:"count"`
34+
}
35+
36+
// APIResponsiveness is a struct for managing multiple API calls.
37+
type APIResponsiveness struct {
38+
APICalls []APICall `json:"apicalls"`
39+
}
40+
41+
// SummaryKind returns the summary of API responsiveness.
42+
func (a *APIResponsiveness) SummaryKind() string {
43+
return "APIResponsiveness"
44+
}
45+
46+
// PrintHumanReadable returns metrics with JSON format.
47+
func (a *APIResponsiveness) PrintHumanReadable() string {
48+
return PrettyPrintJSON(a)
49+
}
50+
51+
// PrintJSON returns metrics of PerfData(50, 90 and 99th percentiles) with JSON format.
52+
func (a *APIResponsiveness) PrintJSON() string {
53+
return PrettyPrintJSON(APICallToPerfData(a))
54+
}
55+
56+
func (a *APIResponsiveness) Len() int { return len(a.APICalls) }
57+
func (a *APIResponsiveness) Swap(i, j int) {
58+
a.APICalls[i], a.APICalls[j] = a.APICalls[j], a.APICalls[i]
59+
}
60+
func (a *APIResponsiveness) Less(i, j int) bool {
61+
return a.APICalls[i].Latency.Perc99 < a.APICalls[j].Latency.Perc99
62+
}
63+
64+
// Set request latency for a particular quantile in the APICall metric entry (creating one if necessary).
65+
// 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
66+
// Only 0.5, 0.9 and 0.99 quantiles are supported.
67+
func (a *APIResponsiveness) addMetricRequestLatency(resource, subresource, verb, scope string, quantile float64, latency time.Duration) {
68+
for i, apicall := range a.APICalls {
69+
if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb && apicall.Scope == scope {
70+
a.APICalls[i] = setQuantileAPICall(apicall, quantile, latency)
71+
return
72+
}
73+
}
74+
apicall := setQuantileAPICall(APICall{Resource: resource, Subresource: subresource, Verb: verb, Scope: scope}, quantile, latency)
75+
a.APICalls = append(a.APICalls, apicall)
76+
}
77+
78+
// 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
79+
// Only 0.5, 0.9 and 0.99 quantiles are supported.
80+
func setQuantileAPICall(apicall APICall, quantile float64, latency time.Duration) APICall {
81+
setQuantile(&apicall.Latency, quantile, latency)
82+
return apicall
83+
}
84+
85+
// Only 0.5, 0.9 and 0.99 quantiles are supported.
86+
func setQuantile(metric *LatencyMetric, quantile float64, latency time.Duration) {
87+
switch quantile {
88+
case 0.5:
89+
metric.Perc50 = latency
90+
case 0.9:
91+
metric.Perc90 = latency
92+
case 0.99:
93+
metric.Perc99 = latency
94+
}
95+
}
96+
97+
// Add request count to the APICall metric entry (creating one if necessary).
98+
func (a *APIResponsiveness) addMetricRequestCount(resource, subresource, verb, scope string, count int) {
99+
for i, apicall := range a.APICalls {
100+
if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb && apicall.Scope == scope {
101+
a.APICalls[i].Count += count
102+
return
103+
}
104+
}
105+
apicall := APICall{Resource: resource, Subresource: subresource, Verb: verb, Count: count, Scope: scope}
106+
a.APICalls = append(a.APICalls, apicall)
107+
}
108+
109+
// currentAPICallMetricsVersion is the current apicall performance metrics version. We should
110+
// bump up the version each time we make incompatible change to the metrics.
111+
const currentAPICallMetricsVersion = "v1"
112+
113+
// APICallToPerfData transforms APIResponsiveness to PerfData.
114+
func APICallToPerfData(apicalls *APIResponsiveness) *e2eperftype.PerfData {
115+
perfData := &e2eperftype.PerfData{Version: currentAPICallMetricsVersion}
116+
for _, apicall := range apicalls.APICalls {
117+
item := e2eperftype.DataItem{
118+
Data: map[string]float64{
119+
"Perc50": float64(apicall.Latency.Perc50) / 1000000, // us -> ms
120+
"Perc90": float64(apicall.Latency.Perc90) / 1000000,
121+
"Perc99": float64(apicall.Latency.Perc99) / 1000000,
122+
},
123+
Unit: "ms",
124+
Labels: map[string]string{
125+
"Verb": apicall.Verb,
126+
"Resource": apicall.Resource,
127+
"Subresource": apicall.Subresource,
128+
"Scope": apicall.Scope,
129+
"Count": fmt.Sprintf("%v", apicall.Count),
130+
},
131+
}
132+
perfData.DataItems = append(perfData.DataItems, item)
133+
}
134+
return perfData
135+
}

0 commit comments

Comments
 (0)