Skip to content

Commit b7a7070

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1942 from bavarianbidi/prefix_gvk_labels_in_customresourcemonitoring
prefix GVK labels in CustomResourceMonitoring
2 parents 559bb28 + 488c0bc commit b7a7070

File tree

6 files changed

+237
-27
lines changed

6 files changed

+237
-27
lines changed

‎docs/customresourcestate-metrics.md

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,41 @@ spec:
4949
- --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,foos,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments,verticalpodautoscalers
5050
```
5151
52-
NOTE: The `group`, `version`, and `kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field.
52+
It's also possible to configure kube-state-metrics to run in a `custom-resource-mode` only. In addition to specifying one of `--custom-resource-state-config*` flags, you could set `--custom-resource-state-only` to `true`.
53+
With this configuration only the known custom resources configured in `--custom-resource-state-config*` will be taken into account by kube-state-metrics.
54+
55+
```yaml
56+
apiVersion: apps/v1
57+
kind: Deployment
58+
metadata:
59+
name: kube-state-metrics
60+
namespace: kube-system
61+
spec:
62+
template:
63+
spec:
64+
containers:
65+
- name: kube-state-metrics
66+
args:
67+
- --custom-resource-state-config
68+
# in YAML files, | allows a multi-line string to be passed as a flag value
69+
# see https://yaml-multiline.info
70+
- |
71+
spec:
72+
resources:
73+
- groupVersionKind:
74+
group: myteam.io
75+
version: "v1"
76+
kind: Foo
77+
metrics:
78+
- name: active_count
79+
help: "Count of active Foo"
80+
each:
81+
type: Gauge
82+
...
83+
- --custom-resource-state-only=true
84+
```
85+
86+
NOTE: The `customresource_group`, `customresource_version`, and `customresource_kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field.
5387

5488
### Examples
5589

@@ -117,7 +151,7 @@ spec:
117151
Produces the metric:
118152

119153
```prometheus
120-
kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
154+
kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
121155
```
122156

123157
#### Multiple Metrics/Kitchen Sink
@@ -169,8 +203,8 @@ spec:
169203
Produces the following metrics:
170204

171205
```prometheus
172-
kube_crd_ready_count{group="myteam.io", kind="Foo", version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2
173-
kube_crd_ready_count{group="myteam.io", kind="Foo", version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4
206+
kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2
207+
kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4
174208
```
175209

176210
### Metric types
@@ -205,7 +239,7 @@ spec:
205239
Produces the metric:
206240

207241
```prometheus
208-
kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
242+
kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
209243
```
210244

211245
#### StateSet
@@ -237,9 +271,9 @@ The value will be 1, if the value matches the one in list.
237271
Produces the metric:
238272

239273
```prometheus
240-
kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Pending"} 1
241-
kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Bar"} 0
242-
kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Baz"} 0
274+
kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Pending"} 1
275+
kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Bar"} 0
276+
kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Baz"} 0
243277
```
244278

245279
#### Info
@@ -269,7 +303,7 @@ spec:
269303
Produces the metric:
270304

271305
```prometheus
272-
kube_crd_version{group="myteam.io", kind="Foo", version="v1", version="v1.2.3"} 1
306+
kube_customresource_version{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", version="v1.2.3"} 1
273307
```
274308

275309
### Naming
@@ -291,7 +325,7 @@ spec:
291325

292326
Produces:
293327
```prometheus
294-
myteam_foos_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
328+
myteam_foos_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
295329
```
296330

297331
To omit namespace and/or subsystem altogether, set them to the empty string:
@@ -309,7 +343,7 @@ spec:
309343

310344
Produces:
311345
```prometheus
312-
uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
346+
uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
313347
```
314348

315349
### Logging

‎docs/verticalpodautoscaler-metrics.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ spec:
3030
gauge:
3131
path: [metadata, annotations]
3232
# This will output the following metric:
33-
# HELP kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations Kubernetes annotations converted to Prometheus labels.
34-
# TYPE kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations gauge
35-
# kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations{namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123
33+
# HELP kube_customresource_autoscaling_annotations Kubernetes annotations converted to Prometheus labels.
34+
# TYPE kube_customresource_autoscaling_annotations gauge
35+
# kube_customresource_autoscaling_annotations{customresource_group="autoscaling.k8s.io", customresource_kind="VerticalPodAutoscaler", customresource_version="v1", namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123
3636
```
3737
PS. The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`).
3838
***

‎pkg/customresourcestate/config.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ import (
2626
"k8s.io/kube-state-metrics/v2/pkg/customresource"
2727
)
2828

29+
// customResourceState is used to prefix the auto-generated GVK labels as well as an appendix for the metric itself
30+
// if no custom metric name is defined
31+
const customResourceState string = "customresource"
32+
2933
// Metrics is the top level configuration object.
3034
type Metrics struct {
3135
Spec MetricsSpec `yaml:"spec" json:"spec"`
@@ -64,7 +68,7 @@ type Resource struct {
6468
func (r Resource) GetMetricNamePrefix() string {
6569
p := r.MetricNamePrefix
6670
if p == nil {
67-
return "kube_crd"
71+
return "kube_" + customResourceState
6872
}
6973
return *p
7074
}

‎pkg/customresourcestate/custom_resource_metrics_test.go

Lines changed: 180 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,87 @@ limitations under the License.
1717
package customresourcestate
1818

1919
import (
20+
"encoding/json"
21+
"reflect"
2022
"testing"
23+
24+
"k8s.io/apimachinery/pkg/runtime/schema"
25+
"k8s.io/utils/pointer"
2126
)
2227

2328
func TestNewCustomResourceMetrics(t *testing.T) {
29+
2430
tests := []struct {
25-
r Resource
26-
wantErr bool
27-
name string
31+
r Resource
32+
wantErr bool
33+
wantResult *customResourceMetrics
34+
name string
2835
}{
2936
{
3037
// https://github.com/kubernetes/kube-state-metrics/issues/1886
31-
name: "dynamic metric type (not just hardcoded to gauge)",
38+
name: "cr metric with dynamic metric type",
39+
r: Resource{
40+
GroupVersionKind: GroupVersionKind{
41+
Group: "apps",
42+
Version: "v1",
43+
Kind: "Deployment",
44+
},
45+
Labels: Labels{
46+
LabelsFromPath: map[string][]string{
47+
"name": {"metadata", "name"},
48+
},
49+
CommonLabels: map[string]string{
50+
"hello": "world",
51+
},
52+
},
53+
Metrics: []Generator{
54+
{
55+
Name: "test_metrics",
56+
Help: "metrics for testing",
57+
Each: Metric{
58+
Type: MetricTypeInfo,
59+
Info: &MetricInfo{
60+
MetricMeta: MetricMeta{
61+
Path: []string{
62+
"metadata",
63+
"annotations",
64+
},
65+
},
66+
LabelFromKey: "test",
67+
},
68+
},
69+
},
70+
},
71+
},
72+
wantErr: false,
73+
wantResult: &customResourceMetrics{
74+
MetricNamePrefix: "kube_customresource",
75+
GroupVersionKind: schema.GroupVersionKind{
76+
Group: "apps",
77+
Version: "v1",
78+
Kind: "Deployment",
79+
},
80+
ResourceName: "deployments",
81+
Families: []compiledFamily{
82+
{
83+
Name: "kube_customresource_test_metrics",
84+
Help: "metrics for testing",
85+
Each: &compiledInfo{},
86+
Labels: map[string]string{
87+
"customresource_group": "apps",
88+
"customresource_kind": "Deployment",
89+
"customresource_version": "v1",
90+
"hello": "world",
91+
},
92+
LabelFromPath: map[string]valuePath{
93+
"name": mustCompilePath(t, "metadata", "name"),
94+
},
95+
},
96+
},
97+
},
98+
},
99+
{
100+
name: "cr metric with custom prefix",
32101
r: Resource{
33102
GroupVersionKind: GroupVersionKind{
34103
Group: "apps",
@@ -39,6 +108,9 @@ func TestNewCustomResourceMetrics(t *testing.T) {
39108
LabelsFromPath: map[string][]string{
40109
"name": {"metadata", "name"},
41110
},
111+
CommonLabels: map[string]string{
112+
"hello": "world",
113+
},
42114
},
43115
Metrics: []Generator{
44116
{
@@ -58,18 +130,118 @@ func TestNewCustomResourceMetrics(t *testing.T) {
58130
},
59131
},
60132
},
133+
MetricNamePrefix: pointer.String("apps_deployment"),
61134
},
62135
wantErr: false,
136+
wantResult: &customResourceMetrics{
137+
MetricNamePrefix: "apps_deployment",
138+
GroupVersionKind: schema.GroupVersionKind{
139+
Group: "apps",
140+
Version: "v1",
141+
Kind: "Deployment",
142+
},
143+
ResourceName: "deployments",
144+
Families: []compiledFamily{
145+
{
146+
Name: "apps_deployment_test_metrics",
147+
Help: "metrics for testing",
148+
Each: &compiledInfo{},
149+
Labels: map[string]string{
150+
"customresource_group": "apps",
151+
"customresource_kind": "Deployment",
152+
"customresource_version": "v1",
153+
"hello": "world",
154+
},
155+
LabelFromPath: map[string]valuePath{
156+
"name": mustCompilePath(t, "metadata", "name"),
157+
},
158+
},
159+
},
160+
},
161+
},
162+
{
163+
name: "cr metric with custom prefix - expect error",
164+
r: Resource{
165+
GroupVersionKind: GroupVersionKind{
166+
Group: "apps",
167+
Version: "v1",
168+
Kind: "Deployment",
169+
},
170+
Labels: Labels{
171+
LabelsFromPath: map[string][]string{
172+
"name": {"metadata", "name"},
173+
},
174+
CommonLabels: map[string]string{
175+
"hello": "world",
176+
},
177+
},
178+
Metrics: []Generator{
179+
{
180+
Name: "test_metrics",
181+
Help: "metrics for testing",
182+
Each: Metric{
183+
Type: MetricTypeInfo,
184+
Info: &MetricInfo{
185+
MetricMeta: MetricMeta{
186+
Path: []string{
187+
"metadata",
188+
"annotations",
189+
},
190+
},
191+
LabelFromKey: "test",
192+
},
193+
},
194+
},
195+
},
196+
MetricNamePrefix: pointer.String("apps_deployment"),
197+
},
198+
wantErr: true,
199+
wantResult: &customResourceMetrics{
200+
GroupVersionKind: schema.GroupVersionKind{
201+
Group: "apps",
202+
Version: "v1",
203+
Kind: "Deployment",
204+
},
205+
ResourceName: "deployments",
206+
Families: []compiledFamily{
207+
{
208+
Name: "apps_deployment_test_metrics",
209+
Help: "metrics for testing",
210+
Each: &compiledInfo{},
211+
Labels: map[string]string{
212+
"customresource_group": "apps",
213+
"customresource_kind": "Deployment",
214+
"customresource_version": "v1",
215+
"hello": "world",
216+
},
217+
LabelFromPath: map[string]valuePath{
218+
"name": mustCompilePath(t, "metadata", "name"),
219+
},
220+
},
221+
},
222+
},
63223
},
64224
}
65225

66226
for _, tt := range tests {
67227
t.Run(tt.name, func(t *testing.T) {
68228
v, err := NewCustomResourceMetrics(tt.r)
69-
expectedError := v.(*customResourceMetrics).Families[0].Each.Type() != "info"
70-
if (err != nil) != tt.wantErr || expectedError {
71-
t.Errorf("NewCustomResourceMetrics() error = %v, wantErr %v", err, tt.wantErr)
72-
return
229+
if err != nil {
230+
t.Errorf(err.Error())
231+
}
232+
233+
// convert to JSON for easier nil comparison
234+
ttWantJSON, _ := json.Marshal(tt.wantResult)
235+
customResourceMetricJSON, _ := json.Marshal(v.(*customResourceMetrics))
236+
237+
if !tt.wantErr && !reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) {
238+
t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr)
239+
t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult)
240+
}
241+
242+
if tt.wantErr && reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) {
243+
t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr)
244+
t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult)
73245
}
74246
})
75247
}

‎pkg/customresourcestate/registry_factory.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ func compile(resource Resource) ([]compiledFamily, error) {
3838
if resource.CommonLabels == nil {
3939
resource.CommonLabels = map[string]string{}
4040
}
41-
resource.CommonLabels["group"] = resource.GroupVersionKind.Group
42-
resource.CommonLabels["version"] = resource.GroupVersionKind.Version
43-
resource.CommonLabels["kind"] = resource.GroupVersionKind.Kind
41+
resource.CommonLabels[customResourceState+"_group"] = resource.GroupVersionKind.Group
42+
resource.CommonLabels[customResourceState+"_version"] = resource.GroupVersionKind.Version
43+
resource.CommonLabels[customResourceState+"_kind"] = resource.GroupVersionKind.Kind
4444
for _, f := range resource.Metrics {
4545
family, err := compileFamily(f, resource)
4646
if err != nil {

‎pkg/customresourcestate/registry_factory_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ func Test_fullName(t *testing.T) {
333333
resource: r(nil),
334334
f: count,
335335
},
336-
want: "kube_crd_count",
336+
want: "kube_customresource_count",
337337
},
338338
{
339339
name: "no prefix",

0 commit comments

Comments
 (0)