Skip to content

Commit f645a68

Browse files
chrischdichrischdi
committed
metrics: add generator for kube-state-metrics customresource configuration
1 parent 8208058 commit f645a68

21 files changed

+1691
-0
lines changed

‎cmd/controller-gen/main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"sigs.k8s.io/controller-tools/pkg/genall/help"
3131
prettyhelp "sigs.k8s.io/controller-tools/pkg/genall/help/pretty"
3232
"sigs.k8s.io/controller-tools/pkg/markers"
33+
"sigs.k8s.io/controller-tools/pkg/metrics"
3334
"sigs.k8s.io/controller-tools/pkg/rbac"
3435
"sigs.k8s.io/controller-tools/pkg/schemapatcher"
3536
"sigs.k8s.io/controller-tools/pkg/version"
@@ -53,6 +54,7 @@ var (
5354
"object": deepcopy.Generator{},
5455
"webhook": webhook.Generator{},
5556
"schemapatch": schemapatcher.Generator{},
57+
"metrics": metrics.Generator{},
5658
}
5759

5860
// allOutputRules defines the list of all known output rules, giving

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ require (
1616
k8s.io/api v0.31.0
1717
k8s.io/apiextensions-apiserver v0.31.0
1818
k8s.io/apimachinery v0.31.0
19+
k8s.io/client-go v0.31.0
1920
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
2021
sigs.k8s.io/yaml v1.4.0
2122
)

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24
179179
k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk=
180180
k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc=
181181
k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
182+
k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=
183+
k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=
182184
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
183185
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
184186
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=

‎pkg/metrics/generator.go

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package metrics contain libraries for generating custom resource metrics configurations
18+
// for kube-state-metrics from metrics markers in Go source files.
19+
package metrics
20+
21+
import (
22+
"fmt"
23+
"sort"
24+
25+
"sigs.k8s.io/controller-tools/pkg/crd"
26+
"sigs.k8s.io/controller-tools/pkg/genall"
27+
"sigs.k8s.io/controller-tools/pkg/loader"
28+
ctrlmarkers "sigs.k8s.io/controller-tools/pkg/markers"
29+
30+
customresourcestate "sigs.k8s.io/controller-tools/pkg/metrics/internal/config"
31+
"sigs.k8s.io/controller-tools/pkg/metrics/markers"
32+
)
33+
34+
// Generator generates kube-state-metrics custom resource configuration files.
35+
type Generator struct{}
36+
37+
var _ genall.Generator = &Generator{}
38+
var _ genall.NeedsTypeChecking = &Generator{}
39+
40+
// RegisterMarkers registers all markers needed by this Generator
41+
// into the given registry.
42+
func (g Generator) RegisterMarkers(into *ctrlmarkers.Registry) error {
43+
for _, m := range markers.MarkerDefinitions {
44+
if err := m.Register(into); err != nil {
45+
return err
46+
}
47+
}
48+
49+
return nil
50+
}
51+
52+
// Generate generates artifacts produced by this marker.
53+
// It's called after RegisterMarkers has been called.
54+
func (g Generator) Generate(ctx *genall.GenerationContext) error {
55+
// Create the parser which is specific to the metric generator.
56+
parser := newParser(
57+
&crd.Parser{
58+
Collector: ctx.Collector,
59+
Checker: ctx.Checker,
60+
},
61+
)
62+
63+
// Loop over all passed packages.
64+
for _, pkg := range ctx.Roots {
65+
// skip packages which don't import metav1 because they can't define a CRD without meta v1.
66+
metav1 := pkg.Imports()["k8s.io/apimachinery/pkg/apis/meta/v1"]
67+
if metav1 == nil {
68+
continue
69+
}
70+
71+
// parse the given package to feed crd.FindKubeKinds with Kubernetes Objects.
72+
parser.NeedPackage(pkg)
73+
74+
kubeKinds := crd.FindKubeKinds(parser.Parser, metav1)
75+
if len(kubeKinds) == 0 {
76+
// no objects in the roots
77+
return nil
78+
}
79+
80+
// Create metrics for all Custom Resources in this package.
81+
// This creates the customresourcestate.Resource object which contains all metric
82+
// definitions for the Custom Resource, if it is part of the package.
83+
for _, gv := range kubeKinds {
84+
if err := parser.NeedResourceFor(pkg, gv); err != nil {
85+
return err
86+
}
87+
}
88+
}
89+
90+
// Initialize empty customresourcestate configuration file and fill it with the
91+
// customresourcestate.Resource objects from the parser.
92+
metrics := customresourcestate.Metrics{
93+
Spec: customresourcestate.MetricsSpec{
94+
Resources: []customresourcestate.Resource{},
95+
},
96+
}
97+
98+
for _, resource := range parser.CustomResourceStates {
99+
if resource == nil {
100+
continue
101+
}
102+
if len(resource.Metrics) > 0 {
103+
// Sort the metrics to get a deterministic output.
104+
sort.Slice(resource.Metrics, func(i, j int) bool {
105+
return resource.Metrics[i].Name < resource.Metrics[j].Name
106+
})
107+
108+
metrics.Spec.Resources = append(metrics.Spec.Resources, *resource)
109+
}
110+
}
111+
112+
// Sort the resources by GVK to get a deterministic output.
113+
sort.Slice(metrics.Spec.Resources, func(i, j int) bool {
114+
a := metrics.Spec.Resources[i].GroupVersionKind.String()
115+
b := metrics.Spec.Resources[j].GroupVersionKind.String()
116+
return a < b
117+
})
118+
119+
// Write the rendered yaml to the context which will result in stdout.
120+
virtualFilePath := "metrics.yaml"
121+
if err := ctx.WriteYAML(virtualFilePath, "", []interface{}{metrics}, genall.WithTransform(addCustomResourceStateKind)); err != nil {
122+
return fmt.Errorf("WriteYAML to %s: %w", virtualFilePath, err)
123+
}
124+
125+
return nil
126+
}
127+
128+
// CheckFilter indicates the loader.NodeFilter (if any) that should be used
129+
// to prune out unused types/packages when type-checking (nodes for which
130+
// the filter returns true are considered "interesting"). This filter acts
131+
// as a baseline -- all types the pass through this filter will be checked,
132+
// but more than that may also be checked due to other generators' filters.
133+
func (Generator) CheckFilter() loader.NodeFilter {
134+
// Re-use controller-tools filter to filter out unrelated nodes that aren't used
135+
// in CRD generation, like interfaces and struct fields without JSON tag.
136+
return crd.Generator{}.CheckFilter()
137+
}
138+
139+
// addCustomResourceStateKind adds the correct kind because we don't have a correct
140+
// kubernetes-style object as configuration definition.
141+
func addCustomResourceStateKind(obj map[string]interface{}) error {
142+
obj["kind"] = "CustomResourceStateMetrics"
143+
return nil
144+
}

‎pkg/metrics/internal/config/config.go

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package config
18+
19+
import (
20+
"fmt"
21+
)
22+
23+
// Metrics is the top level configuration object.
24+
type Metrics struct {
25+
Spec MetricsSpec `yaml:"spec" json:"spec"`
26+
}
27+
28+
// MetricsSpec is the configuration describing the custom resource state metrics to generate.
29+
type MetricsSpec struct {
30+
// Resources is the list of custom resources to be monitored. A resource with the same GroupVersionKind may appear
31+
// multiple times (e.g., to customize the namespace or subsystem,) but will incur additional overhead.
32+
Resources []Resource `yaml:"resources" json:"resources"`
33+
}
34+
35+
// Resource configures a custom resource for metric generation.
36+
type Resource struct {
37+
// MetricNamePrefix defines a prefix for all metrics of the resource.
38+
// If set to "", no prefix will be added.
39+
// Example: If set to "foo", MetricNamePrefix will be "foo_<metric>".
40+
MetricNamePrefix *string `yaml:"metricNamePrefix" json:"metricNamePrefix"`
41+
42+
// GroupVersionKind of the custom resource to be monitored.
43+
GroupVersionKind GroupVersionKind `yaml:"groupVersionKind" json:"groupVersionKind"`
44+
45+
// Labels are added to all metrics. If the same key is used in a metric, the value from the metric will overwrite the value here.
46+
Labels `yaml:",inline" json:",inline"`
47+
48+
// Metrics are the custom resource fields to be collected.
49+
Metrics []Generator `yaml:"metrics" json:"metrics"`
50+
// ErrorLogV defines the verbosity threshold for errors logged for this resource.
51+
ErrorLogV int32 `yaml:"errorLogV" json:"errorLogV"`
52+
53+
// ResourcePlural sets the plural name of the resource. Defaults to the plural version of the Kind according to flect.Pluralize.
54+
ResourcePlural string `yaml:"resourcePlural" json:"resourcePlural"`
55+
}
56+
57+
// GroupVersionKind is the Kubernetes group, version, and kind of a resource.
58+
type GroupVersionKind struct {
59+
Group string `yaml:"group" json:"group"`
60+
Version string `yaml:"version" json:"version"`
61+
Kind string `yaml:"kind" json:"kind"`
62+
}
63+
64+
func (gvk GroupVersionKind) String() string {
65+
return fmt.Sprintf("%s_%s_%s", gvk.Group, gvk.Version, gvk.Kind)
66+
}
67+
68+
// Labels is common configuration of labels to add to metrics.
69+
type Labels struct {
70+
// CommonLabels are added to all metrics.
71+
CommonLabels map[string]string `yaml:"commonLabels,omitempty" json:"commonLabels,omitempty"`
72+
// LabelsFromPath adds additional labels where the value is taken from a field in the resource.
73+
LabelsFromPath map[string][]string `yaml:"labelsFromPath,omitempty" json:"labelsFromPath,omitempty"`
74+
}
75+
76+
// Generator describes a unique metric name.
77+
type Generator struct {
78+
// Name of the metric. Subject to prefixing based on the configuration of the Resource.
79+
Name string `yaml:"name" json:"name"`
80+
// Help text for the metric.
81+
Help string `yaml:"help" json:"help"`
82+
// Each targets a value or values from the resource.
83+
Each Metric `yaml:"each" json:"each"`
84+
85+
// Labels are added to all metrics. Labels from Each will overwrite these if using the same key.
86+
Labels `yaml:",inline" json:",inline"` // json will inline because it is already tagged
87+
// ErrorLogV defines the verbosity threshold for errors logged for this metric. Must be non-zero to override the resource setting.
88+
ErrorLogV int32 `yaml:"errorLogV,omitempty" json:"errorLogV,omitempty"`
89+
}
90+
91+
// Metric defines a metric to expose.
92+
// +union
93+
type Metric struct {
94+
// Type defines the type of the metric.
95+
// +unionDiscriminator
96+
Type MetricType `yaml:"type" json:"type"`
97+
98+
// Gauge defines a gauge metric.
99+
// +optional
100+
Gauge *MetricGauge `yaml:"gauge,omitempty" json:"gauge,omitempty"`
101+
// StateSet defines a state set metric.
102+
// +optional
103+
StateSet *MetricStateSet `yaml:"stateSet,omitempty" json:"stateSet,omitempty"`
104+
// Info defines an info metric.
105+
// +optional
106+
Info *MetricInfo `yaml:"info,omitempty" json:"info,omitempty"`
107+
}
108+
109+
// ConfigDecoder is for use with FromConfig.
110+
type ConfigDecoder interface {
111+
Decode(v interface{}) (err error)
112+
}

‎pkg/metrics/internal/config/config_metrics_types.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package config
18+
19+
// MetricMeta are variables which may used for any metric type.
20+
type MetricMeta struct {
21+
// LabelsFromPath adds additional labels where the value of the label is taken from a field under Path.
22+
LabelsFromPath map[string][]string `yaml:"labelsFromPath,omitempty" json:"labelsFromPath,omitempty"`
23+
// Path is the path to to generate metric(s) for.
24+
Path []string `yaml:"path" json:"path"`
25+
}
26+
27+
// MetricGauge targets a Path that may be a single value, array, or object. Arrays and objects will generate a metric per element.
28+
// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge
29+
type MetricGauge struct {
30+
MetricMeta `yaml:",inline" json:",inline"`
31+
32+
// ValueFrom is the path to a numeric field under Path that will be the metric value.
33+
ValueFrom []string `yaml:"valueFrom" json:"valueFrom"`
34+
// LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key.
35+
LabelFromKey string `yaml:"labelFromKey" json:"labelFromKey"`
36+
// NilIsZero indicates that if a value is nil it will be treated as zero value.
37+
NilIsZero bool `yaml:"nilIsZero" json:"nilIsZero"`
38+
}
39+
40+
// MetricInfo is a metric which is used to expose textual information.
41+
// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info
42+
type MetricInfo struct {
43+
MetricMeta `yaml:",inline" json:",inline"`
44+
// LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key.
45+
LabelFromKey string `yaml:"labelFromKey" json:"labelFromKey"`
46+
}
47+
48+
// MetricStateSet is a metric which represent a series of related boolean values, also called a bitset.
49+
// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset
50+
type MetricStateSet struct {
51+
MetricMeta `yaml:",inline" json:",inline"`
52+
53+
// List is the list of values to expose a value for.
54+
List []string `yaml:"list" json:"list"`
55+
// LabelName is the key of the label which is used for each entry in List to expose the value.
56+
LabelName string `yaml:"labelName" json:"labelName"`
57+
// ValueFrom is the subpath to compare the list to.
58+
ValueFrom []string `yaml:"valueFrom" json:"valueFrom"`
59+
}

‎pkg/metrics/internal/config/doc.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// config contains a copy of the types from k8s.io/kube-state-metrics/pkg/customresourcestate.
18+
// It is currently based on kube-state-metrics v2.13.0.
19+
// The following modifications got applied:
20+
// For `config.go`:
21+
// * Rename the package to `config`.
22+
// * Drop `const customResourceState`.
23+
// * Drop all functions, only preserve structs.
24+
// * Use `int32` instead of `klog.Level`.
25+
// * Use `MetricType` instead of `metric.Type`
26+
// * Add `omitempty` to:
27+
// - `Labels.CommonLabels`
28+
// - `Labels.LabelsFromPath`
29+
// - `Generator.ErrorLogV`
30+
// - `Metric.Gauge`
31+
// - `Metric.StateSet`
32+
// - `Metric.Info`
33+
//
34+
// For `config_metrics_types.go`:
35+
// * Rename the package to `config`.
36+
// * Add `omitempty` to:
37+
// - `MetricMeta.LabelsFromPath
38+
package config

0 commit comments

Comments
 (0)