Skip to content

Commit 620fa75

Browse files
authored
feat: add k8s custom metrics collector (#1174)
Collector that collects custom k8s metrics from custom.metrics.k8s.io/v1beta1/ and saves them in the bundle under the /metrics directory
1 parent 60d5b68 commit 620fa75

13 files changed

+483
-0
lines changed

config/crds/troubleshoot.sh_collectors.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,36 @@ spec:
245245
- image
246246
- namespace
247247
type: object
248+
customMetrics:
249+
properties:
250+
collectorName:
251+
type: string
252+
exclude:
253+
type: BoolString
254+
metricRequests:
255+
items:
256+
description: MetricRequest the details of the MetricValuesList
257+
to be retrieved
258+
properties:
259+
namespace:
260+
description: Namespace for which to collect the metric
261+
values, empty for non-namespaces resources.
262+
type: string
263+
objectName:
264+
description: ObjectName for which to collect metric
265+
values, all resources when empty. Note that for
266+
namespaced resources a Namespace has to be supplied
267+
regardless.
268+
type: string
269+
resourceMetricName:
270+
description: ResourceMetricName name of the MetricValueList
271+
as per the APIResourceList from custom.metrics.k8s.io/v1beta1
272+
type: string
273+
required:
274+
- resourceMetricName
275+
type: object
276+
type: array
277+
type: object
248278
data:
249279
properties:
250280
collectorName:

config/crds/troubleshoot.sh_preflights.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,6 +1740,36 @@ spec:
17401740
- image
17411741
- namespace
17421742
type: object
1743+
customMetrics:
1744+
properties:
1745+
collectorName:
1746+
type: string
1747+
exclude:
1748+
type: BoolString
1749+
metricRequests:
1750+
items:
1751+
description: MetricRequest the details of the MetricValuesList
1752+
to be retrieved
1753+
properties:
1754+
namespace:
1755+
description: Namespace for which to collect the metric
1756+
values, empty for non-namespaces resources.
1757+
type: string
1758+
objectName:
1759+
description: ObjectName for which to collect metric
1760+
values, all resources when empty. Note that for
1761+
namespaced resources a Namespace has to be supplied
1762+
regardless.
1763+
type: string
1764+
resourceMetricName:
1765+
description: ResourceMetricName name of the MetricValueList
1766+
as per the APIResourceList from custom.metrics.k8s.io/v1beta1
1767+
type: string
1768+
required:
1769+
- resourceMetricName
1770+
type: object
1771+
type: array
1772+
type: object
17431773
data:
17441774
properties:
17451775
collectorName:

config/crds/troubleshoot.sh_supportbundles.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1771,6 +1771,36 @@ spec:
17711771
- image
17721772
- namespace
17731773
type: object
1774+
customMetrics:
1775+
properties:
1776+
collectorName:
1777+
type: string
1778+
exclude:
1779+
type: BoolString
1780+
metricRequests:
1781+
items:
1782+
description: MetricRequest the details of the MetricValuesList
1783+
to be retrieved
1784+
properties:
1785+
namespace:
1786+
description: Namespace for which to collect the metric
1787+
values, empty for non-namespaces resources.
1788+
type: string
1789+
objectName:
1790+
description: ObjectName for which to collect metric
1791+
values, all resources when empty. Note that for
1792+
namespaced resources a Namespace has to be supplied
1793+
regardless.
1794+
type: string
1795+
resourceMetricName:
1796+
description: ResourceMetricName name of the MetricValueList
1797+
as per the APIResourceList from custom.metrics.k8s.io/v1beta1
1798+
type: string
1799+
required:
1800+
- resourceMetricName
1801+
type: object
1802+
type: array
1803+
type: object
17741804
data:
17751805
properties:
17761806
collectorName:

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ require (
211211
gopkg.in/ini.v1 v1.67.0 // indirect
212212
gopkg.in/yaml.v3 v3.0.1 // indirect
213213
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
214+
k8s.io/metrics v0.27.2
214215
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
215216
periph.io/x/host/v3 v3.8.2
216217
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1476,6 +1476,8 @@ k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
14761476
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
14771477
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
14781478
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
1479+
k8s.io/metrics v0.27.2 h1:TD6z3dhhN9bgg5YkbTh72bPiC1BsxipBLPBWyC3VQAU=
1480+
k8s.io/metrics v0.27.2/go.mod h1:v3OT7U0DBvoAzWVzGZWQhdV4qsRJWchzs/LeVN8bhW4=
14791481
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
14801482
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
14811483
oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY=

pkg/apis/troubleshoot/v1beta2/collector_shared.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,23 @@ type ClusterResources struct {
2727
IgnoreRBAC bool `json:"ignoreRBAC,omitempty" yaml:"ignoreRBAC"`
2828
}
2929

30+
// MetricRequest the details of the MetricValuesList to be retrieved
31+
type MetricRequest struct {
32+
// Namespace for which to collect the metric values, empty for non-namespaces resources.
33+
Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
34+
// ObjectName for which to collect metric values, all resources when empty.
35+
// Note that for namespaced resources a Namespace has to be supplied regardless.
36+
ObjectName string `json:"objectName,omitempty" yaml:"objectName,omitempty"`
37+
// ResourceMetricName name of the MetricValueList as per the APIResourceList from
38+
// custom.metrics.k8s.io/v1beta1
39+
ResourceMetricName string `json:"resourceMetricName" yaml:"resourceMetricName"`
40+
}
41+
42+
type CustomMetrics struct {
43+
CollectorMeta `json:",inline" yaml:",inline"`
44+
MetricRequests []MetricRequest `json:"metricRequests,omitempty" yaml:"metricRequests,omitempty"`
45+
}
46+
3047
type Secret struct {
3148
CollectorMeta `json:",inline" yaml:",inline"`
3249
Name string `json:"name,omitempty" yaml:"name,omitempty"`
@@ -231,6 +248,7 @@ type Collect struct {
231248
ClusterInfo *ClusterInfo `json:"clusterInfo,omitempty" yaml:"clusterInfo,omitempty"`
232249
ClusterResources *ClusterResources `json:"clusterResources,omitempty" yaml:"clusterResources,omitempty"`
233250
Secret *Secret `json:"secret,omitempty" yaml:"secret,omitempty"`
251+
CustomMetrics *CustomMetrics `json:"customMetrics,omitempty" yaml:"customMetrics,omitempty"`
234252
ConfigMap *ConfigMap `json:"configMap,omitempty" yaml:"configMap,omitempty"`
235253
Logs *Logs `json:"logs,omitempty" yaml:"logs,omitempty"`
236254
Run *Run `json:"run,omitempty" yaml:"run,omitempty"`

pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go

Lines changed: 41 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/collect/collector.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ func GetCollector(collector *troubleshootv1beta2.Collect, bundlePath string, nam
6363
return &CollectClusterInfo{collector.ClusterInfo, bundlePath, namespace, clientConfig, RBACErrors}, true
6464
case collector.ClusterResources != nil:
6565
return &CollectClusterResources{collector.ClusterResources, bundlePath, namespace, clientConfig, RBACErrors}, true
66+
case collector.CustomMetrics != nil:
67+
return &CollectMetrics{collector.CustomMetrics, bundlePath, clientConfig, client, ctx, RBACErrors}, true
6668
case collector.Secret != nil:
6769
return &CollectSecret{collector.Secret, bundlePath, namespace, clientConfig, client, ctx, RBACErrors}, true
6870
case collector.ConfigMap != nil:
@@ -116,6 +118,9 @@ func getCollectorName(c interface{}) string {
116118
collector = "cluster-info"
117119
case *CollectClusterResources:
118120
collector = "cluster-resources"
121+
case *CollectMetrics:
122+
collector = "custom-metrics"
123+
name = v.Collector.CollectorName
119124
case *CollectSecret:
120125
collector = "secret"
121126
name = v.Collector.CollectorName

pkg/collect/k8s_metrics.go

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package collect
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"fmt"
8+
"net/url"
9+
"path/filepath"
10+
"strings"
11+
12+
"github.com/pkg/errors"
13+
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
14+
"k8s.io/client-go/kubernetes"
15+
"k8s.io/client-go/rest"
16+
"k8s.io/klog/v2"
17+
"k8s.io/metrics/pkg/apis/custom_metrics"
18+
)
19+
20+
const (
21+
namespaceSingular = "namespace"
22+
namespacePlural = "namespaces"
23+
urlBase = "/apis/custom.metrics.k8s.io/v1beta1"
24+
metricsErrorFile = "metrics/errors.json"
25+
)
26+
27+
type CollectMetrics struct {
28+
Collector *troubleshootv1beta2.CustomMetrics
29+
BundlePath string
30+
ClientConfig *rest.Config
31+
Client kubernetes.Interface
32+
Context context.Context
33+
RBACErrors
34+
}
35+
36+
func (c *CollectMetrics) Title() string {
37+
return getCollectorName(c)
38+
}
39+
40+
func (c *CollectMetrics) IsExcluded() (bool, error) {
41+
return isExcluded(c.Collector.Exclude)
42+
}
43+
44+
func (c *CollectMetrics) Collect(progressChan chan<- interface{}) (CollectorResult, error) {
45+
output := NewResult()
46+
resultLists := make(map[string][]custom_metrics.MetricValue)
47+
errorsList := make([]string, 0)
48+
for _, metricRequest := range c.Collector.MetricRequests {
49+
klog.V(2).Infof("Getting metric values: %+v\n", metricRequest.ResourceMetricName)
50+
endpoint, metricName, err := constructEndpoint(metricRequest)
51+
if err != nil {
52+
errorsList = append(errorsList, errors.Wrapf(err, "could not construct endpoint for %s", metricRequest.ResourceMetricName).Error())
53+
continue
54+
}
55+
klog.V(2).Infof("Querying: %+v\n", endpoint)
56+
response, err := c.Client.CoreV1().RESTClient().Get().AbsPath(endpoint).DoRaw(c.Context)
57+
if err != nil {
58+
errorsList = append(errorsList, errors.Wrapf(err, "could not query endpoint %s", endpoint).Error())
59+
continue
60+
}
61+
metricsValues := custom_metrics.MetricValueList{}
62+
json.Unmarshal(response, &metricsValues)
63+
// metrics
64+
// |_ <resource_type>
65+
// |_ <metric_name>
66+
// |_ <namespace>.json or <non_namespaced_object>.json
67+
var path []string
68+
for _, item := range metricsValues.Items {
69+
if item.DescribedObject.Namespace != "" {
70+
path = []string{"metrics", item.DescribedObject.Kind, metricName, fmt.Sprintf("%s.json", item.DescribedObject.Namespace)}
71+
} else {
72+
path = []string{"metrics", item.DescribedObject.Kind, metricName, fmt.Sprintf("%s.json", item.DescribedObject.Name)}
73+
}
74+
filePath := filepath.Join(path...)
75+
if _, ok := resultLists[filePath]; !ok {
76+
resultLists[filePath] = make([]custom_metrics.MetricValue, 0)
77+
}
78+
resultLists[filePath] = append(resultLists[filePath], item)
79+
}
80+
}
81+
82+
// Construct output.
83+
for relativePath, list := range resultLists {
84+
payload, err := json.MarshalIndent(list, "", " ")
85+
if err != nil {
86+
klog.V(2).Infof("Could not parse for: %+v\n", relativePath)
87+
errorsList = append(errorsList, errors.Wrapf(err, "could not format readings for %s", relativePath).Error())
88+
}
89+
output.SaveResult(c.BundlePath, relativePath, bytes.NewBuffer(payload))
90+
}
91+
errPayload := marshalErrors(errorsList)
92+
output.SaveResult(c.BundlePath, metricsErrorFile, errPayload)
93+
return output, nil
94+
}
95+
96+
func constructEndpoint(metricRequest troubleshootv1beta2.MetricRequest) (string, string, error) {
97+
metricNameComponents := strings.Split(metricRequest.ResourceMetricName, "/")
98+
if len(metricNameComponents) != 2 {
99+
return "", "", errors.New("wrong metric name format %s")
100+
}
101+
objectType := metricNameComponents[0]
102+
// Namespace related metrics are grouped under singular format "namespace/"
103+
// unlike other resources.
104+
if objectType == namespacePlural {
105+
objectType = namespaceSingular
106+
}
107+
metricName := metricNameComponents[1]
108+
objectSelector := "*"
109+
if metricRequest.ObjectName != "" {
110+
objectSelector = metricRequest.ObjectName
111+
}
112+
var endpoint string
113+
var err error
114+
if metricRequest.Namespace != "" {
115+
// namespaced objects
116+
// endpoint <resource_type>/namespaces/<namespace>/<resrouce_name or *>/<metric>
117+
endpoint, err = url.JoinPath(urlBase, namespacePlural, metricRequest.Namespace, objectType, objectSelector, metricName)
118+
if err != nil {
119+
return "", "", errors.Wrap(err, "could not construct url")
120+
}
121+
} else {
122+
// non-namespaced objects
123+
// endpoint <resource_type>/<resrouce_name or *>/<metric>
124+
endpoint, err = url.JoinPath(urlBase, objectType, objectSelector, metricName)
125+
if err != nil {
126+
return "", "", errors.Wrap(err, "could not construct url")
127+
}
128+
}
129+
return endpoint, metricName, nil
130+
}

0 commit comments

Comments
 (0)