Merge pull request kubernetes#1942 from bavarianbidi/prefix_gvk_labels_in_customresourcemonitoring

k8s-ci-robot · web-flow · commit b7a7070e87ba · 2023-01-12T00:58:54.000-08:00
prefix GVK labels in CustomResourceMonitoring
diff --git a/docs/customresourcestate-metrics.md b/docs/customresourcestate-metrics.md
@@ -49,7 +49,41 @@ spec:
           - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,foos,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments,verticalpodautoscalers
 ```
 
-NOTE: The `group`, `version`, and `kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field.
+It's also possible to configure kube-state-metrics to run in a `custom-resource-mode` only. In addition to specifying one of `--custom-resource-state-config*` flags, you could set `--custom-resource-state-only` to `true`.
+With this configuration only the known custom resources configured in `--custom-resource-state-config*` will be taken into account by kube-state-metrics.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-state-metrics
+        args:
+          - --custom-resource-state-config
+          # in YAML files, | allows a multi-line string to be passed as a flag value
+          # see https://yaml-multiline.info
+          -  |
+              spec:
+                resources:
+                  - groupVersionKind:
+                      group: myteam.io
+                      version: "v1"
+                      kind: Foo
+                    metrics:
+                      - name: active_count
+                        help: "Count of active Foo"
+                        each:
+                          type: Gauge
+                          ...
+          - --custom-resource-state-only=true
+```
+
+NOTE: The `customresource_group`, `customresource_version`, and `customresource_kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field.
 
 ### Examples
 
@@ -117,7 +151,7 @@ spec:
 Produces the metric:
 
 ```prometheus
-kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
+kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
 ```
 
 #### Multiple Metrics/Kitchen Sink
@@ -169,8 +203,8 @@ spec:
 Produces the following metrics:
 
 ```prometheus
-kube_crd_ready_count{group="myteam.io", kind="Foo", version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2
-kube_crd_ready_count{group="myteam.io", kind="Foo", version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4
+kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2
+kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4
 ```
 
 ### Metric types
@@ -205,7 +239,7 @@ spec:
 Produces the metric:
 
 ```prometheus
-kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
+kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
 ```
 
 #### StateSet
@@ -237,9 +271,9 @@ The value will be 1, if the value matches the one in list.
 Produces the metric:
 
 ```prometheus
-kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Pending"} 1
-kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Bar"} 0
-kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Baz"} 0
+kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Pending"} 1
+kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Bar"} 0
+kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Baz"} 0
 ```
 
 #### Info
@@ -269,7 +303,7 @@ spec:
 Produces the metric:
 
 ```prometheus
-kube_crd_version{group="myteam.io", kind="Foo", version="v1", version="v1.2.3"} 1
+kube_customresource_version{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", version="v1.2.3"} 1
 ```
 
 ### Naming
@@ -291,7 +325,7 @@ spec:
 
 Produces:
 ```prometheus
-myteam_foos_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
+myteam_foos_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
 ```
 
 To omit namespace and/or subsystem altogether, set them to the empty string:
@@ -309,7 +343,7 @@ spec:
 
 Produces:
 ```prometheus
-uptime{group="myteam.io", kind="Foo", version="v1"} 43.21
+uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
 ```
 
 ### Logging
diff --git a/docs/verticalpodautoscaler-metrics.md b/docs/verticalpodautoscaler-metrics.md
@@ -30,9 +30,9 @@ spec:
             gauge:
               path: [metadata, annotations]
 # This will output the following metric:
-# HELP kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations Kubernetes annotations converted to Prometheus labels.
-# TYPE kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations gauge
-# kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations{namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123
+# HELP kube_customresource_autoscaling_annotations Kubernetes annotations converted to Prometheus labels.
+# TYPE kube_customresource_autoscaling_annotations gauge
+# kube_customresource_autoscaling_annotations{customresource_group="autoscaling.k8s.io", customresource_kind="VerticalPodAutoscaler", customresource_version="v1", namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123
 ```
 PS. The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`).
 ***
diff --git a/pkg/customresourcestate/config.go b/pkg/customresourcestate/config.go
@@ -26,6 +26,10 @@ import (
 	"k8s.io/kube-state-metrics/v2/pkg/customresource"
 )
 
+// customResourceState is used to prefix the auto-generated GVK labels as well as an appendix for the metric itself
+// if no custom metric name is defined
+const customResourceState string = "customresource"
+
 // Metrics is the top level configuration object.
 type Metrics struct {
 	Spec MetricsSpec `yaml:"spec" json:"spec"`
@@ -64,7 +68,7 @@ type Resource struct {
 func (r Resource) GetMetricNamePrefix() string {
 	p := r.MetricNamePrefix
 	if p == nil {
-		return "kube_crd"
+		return "kube_" + customResourceState
 	}
 	return *p
 }
diff --git a/pkg/customresourcestate/custom_resource_metrics_test.go b/pkg/customresourcestate/custom_resource_metrics_test.go
@@ -17,18 +17,87 @@ limitations under the License.
 package customresourcestate
 
 import (
+	"encoding/json"
+	"reflect"
 	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/utils/pointer"
 )
 
 func TestNewCustomResourceMetrics(t *testing.T) {
+
 	tests := []struct {
-		r       Resource
-		wantErr bool
-		name    string
+		r          Resource
+		wantErr    bool
+		wantResult *customResourceMetrics
+		name       string
 	}{
 		{
 			// https://github.com/kubernetes/kube-state-metrics/issues/1886
-			name: "dynamic metric type (not just hardcoded to gauge)",
+			name: "cr metric with dynamic metric type",
+			r: Resource{
+				GroupVersionKind: GroupVersionKind{
+					Group:   "apps",
+					Version: "v1",
+					Kind:    "Deployment",
+				},
+				Labels: Labels{
+					LabelsFromPath: map[string][]string{
+						"name": {"metadata", "name"},
+					},
+					CommonLabels: map[string]string{
+						"hello": "world",
+					},
+				},
+				Metrics: []Generator{
+					{
+						Name: "test_metrics",
+						Help: "metrics for testing",
+						Each: Metric{
+							Type: MetricTypeInfo,
+							Info: &MetricInfo{
+								MetricMeta: MetricMeta{
+									Path: []string{
+										"metadata",
+										"annotations",
+									},
+								},
+								LabelFromKey: "test",
+							},
+						},
+					},
+				},
+			},
+			wantErr: false,
+			wantResult: &customResourceMetrics{
+				MetricNamePrefix: "kube_customresource",
+				GroupVersionKind: schema.GroupVersionKind{
+					Group:   "apps",
+					Version: "v1",
+					Kind:    "Deployment",
+				},
+				ResourceName: "deployments",
+				Families: []compiledFamily{
+					{
+						Name: "kube_customresource_test_metrics",
+						Help: "metrics for testing",
+						Each: &compiledInfo{},
+						Labels: map[string]string{
+							"customresource_group":   "apps",
+							"customresource_kind":    "Deployment",
+							"customresource_version": "v1",
+							"hello":                  "world",
+						},
+						LabelFromPath: map[string]valuePath{
+							"name": mustCompilePath(t, "metadata", "name"),
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "cr metric with custom prefix",
 			r: Resource{
 				GroupVersionKind: GroupVersionKind{
 					Group:   "apps",
@@ -39,6 +108,9 @@ func TestNewCustomResourceMetrics(t *testing.T) {
 					LabelsFromPath: map[string][]string{
 						"name": {"metadata", "name"},
 					},
+					CommonLabels: map[string]string{
+						"hello": "world",
+					},
 				},
 				Metrics: []Generator{
 					{
@@ -58,18 +130,118 @@ func TestNewCustomResourceMetrics(t *testing.T) {
 						},
 					},
 				},
+				MetricNamePrefix: pointer.String("apps_deployment"),
 			},
 			wantErr: false,
+			wantResult: &customResourceMetrics{
+				MetricNamePrefix: "apps_deployment",
+				GroupVersionKind: schema.GroupVersionKind{
+					Group:   "apps",
+					Version: "v1",
+					Kind:    "Deployment",
+				},
+				ResourceName: "deployments",
+				Families: []compiledFamily{
+					{
+						Name: "apps_deployment_test_metrics",
+						Help: "metrics for testing",
+						Each: &compiledInfo{},
+						Labels: map[string]string{
+							"customresource_group":   "apps",
+							"customresource_kind":    "Deployment",
+							"customresource_version": "v1",
+							"hello":                  "world",
+						},
+						LabelFromPath: map[string]valuePath{
+							"name": mustCompilePath(t, "metadata", "name"),
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "cr metric with custom prefix - expect error",
+			r: Resource{
+				GroupVersionKind: GroupVersionKind{
+					Group:   "apps",
+					Version: "v1",
+					Kind:    "Deployment",
+				},
+				Labels: Labels{
+					LabelsFromPath: map[string][]string{
+						"name": {"metadata", "name"},
+					},
+					CommonLabels: map[string]string{
+						"hello": "world",
+					},
+				},
+				Metrics: []Generator{
+					{
+						Name: "test_metrics",
+						Help: "metrics for testing",
+						Each: Metric{
+							Type: MetricTypeInfo,
+							Info: &MetricInfo{
+								MetricMeta: MetricMeta{
+									Path: []string{
+										"metadata",
+										"annotations",
+									},
+								},
+								LabelFromKey: "test",
+							},
+						},
+					},
+				},
+				MetricNamePrefix: pointer.String("apps_deployment"),
+			},
+			wantErr: true,
+			wantResult: &customResourceMetrics{
+				GroupVersionKind: schema.GroupVersionKind{
+					Group:   "apps",
+					Version: "v1",
+					Kind:    "Deployment",
+				},
+				ResourceName: "deployments",
+				Families: []compiledFamily{
+					{
+						Name: "apps_deployment_test_metrics",
+						Help: "metrics for testing",
+						Each: &compiledInfo{},
+						Labels: map[string]string{
+							"customresource_group":   "apps",
+							"customresource_kind":    "Deployment",
+							"customresource_version": "v1",
+							"hello":                  "world",
+						},
+						LabelFromPath: map[string]valuePath{
+							"name": mustCompilePath(t, "metadata", "name"),
+						},
+					},
+				},
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			v, err := NewCustomResourceMetrics(tt.r)
-			expectedError := v.(*customResourceMetrics).Families[0].Each.Type() != "info"
-			if (err != nil) != tt.wantErr || expectedError {
-				t.Errorf("NewCustomResourceMetrics() error = %v, wantErr %v", err, tt.wantErr)
-				return
+			if err != nil {
+				t.Errorf(err.Error())
+			}
+
+			// convert to JSON for easier nil comparison
+			ttWantJSON, _ := json.Marshal(tt.wantResult)
+			customResourceMetricJSON, _ := json.Marshal(v.(*customResourceMetrics))
+
+			if !tt.wantErr && !reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) {
+				t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr)
+				t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult)
+			}
+
+			if tt.wantErr && reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) {
+				t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr)
+				t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult)
 			}
 		})
 	}
diff --git a/pkg/customresourcestate/registry_factory.go b/pkg/customresourcestate/registry_factory.go
@@ -38,9 +38,9 @@ func compile(resource Resource) ([]compiledFamily, error) {
 	if resource.CommonLabels == nil {
 		resource.CommonLabels = map[string]string{}
 	}
-	resource.CommonLabels["group"] = resource.GroupVersionKind.Group
-	resource.CommonLabels["version"] = resource.GroupVersionKind.Version
-	resource.CommonLabels["kind"] = resource.GroupVersionKind.Kind
+	resource.CommonLabels[customResourceState+"_group"] = resource.GroupVersionKind.Group
+	resource.CommonLabels[customResourceState+"_version"] = resource.GroupVersionKind.Version
+	resource.CommonLabels[customResourceState+"_kind"] = resource.GroupVersionKind.Kind
 	for _, f := range resource.Metrics {
 		family, err := compileFamily(f, resource)
 		if err != nil {
diff --git a/pkg/customresourcestate/registry_factory_test.go b/pkg/customresourcestate/registry_factory_test.go
@@ -333,7 +333,7 @@ func Test_fullName(t *testing.T) {
 				resource: r(nil),
 				f:        count,
 			},
-			want: "kube_crd_count",
+			want: "kube_customresource_count",
 		},
 		{
 			name: "no prefix",

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,10 @@ import (`
`26`	`26`	`"k8s.io/kube-state-metrics/v2/pkg/customresource"`
`27`	`27`	`)`
`28`	`28`
	`29`	`+// customResourceState is used to prefix the auto-generated GVK labels as well as an appendix for the metric itself`
	`30`	`+// if no custom metric name is defined`
	`31`	`+const customResourceState string = "customresource"`
	`32`	`+`
`29`	`33`	`// Metrics is the top level configuration object.`
`30`	`34`	`type Metrics struct {`
`31`	`35`	Spec MetricsSpec `yaml:"spec" json:"spec"`
`@@ -64,7 +68,7 @@ type Resource struct {`
`64`	`68`	`func (r Resource) GetMetricNamePrefix() string {`
`65`	`69`	`p := r.MetricNamePrefix`
`66`	`70`	`if p == nil {`
`67`		`- return "kube_crd"`
	`71`	`+ return "kube_" + customResourceState`
`68`	`72`	`}`
`69`	`73`	`return *p`
`70`	`74`	`}`
Original file line number	Diff line number	Diff line change
`@@ -333,7 +333,7 @@ func Test_fullName(t *testing.T) {`
`333`	`333`	`resource: r(nil),`
`334`	`334`	`f: count,`
`335`	`335`	`},`
`336`		`- want: "kube_crd_count",`
	`336`	`+ want: "kube_customresource_count",`
`337`	`337`	`},`
`338`	`338`	`{`
`339`	`339`	`name: "no prefix",`