diff --git a/cmd/controller-gen/main.go b/cmd/controller-gen/main.go index b421028a9..c3d39621e 100644 --- a/cmd/controller-gen/main.go +++ b/cmd/controller-gen/main.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-tools/pkg/genall/help" prettyhelp "sigs.k8s.io/controller-tools/pkg/genall/help/pretty" "sigs.k8s.io/controller-tools/pkg/markers" + "sigs.k8s.io/controller-tools/pkg/metrics" "sigs.k8s.io/controller-tools/pkg/rbac" "sigs.k8s.io/controller-tools/pkg/schemapatcher" "sigs.k8s.io/controller-tools/pkg/version" @@ -57,6 +58,7 @@ var ( "applyconfiguration": applyconfiguration.Generator{}, "webhook": webhook.Generator{}, "schemapatch": schemapatcher.Generator{}, + "metrics": metrics.Generator{}, } // allOutputRules defines the list of all known output rules, giving diff --git a/go.mod b/go.mod index c8b8821b1..354437500 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( k8s.io/apiextensions-apiserver v0.34.0-rc.1 k8s.io/apimachinery v0.34.0-rc.1 k8s.io/apiserver v0.34.0-rc.1 + k8s.io/client-go v0.34.0-rc.1 k8s.io/code-generator v0.34.0-rc.1 k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 @@ -88,7 +89,6 @@ require ( google.golang.org/protobuf v1.36.6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - k8s.io/client-go v0.34.0-rc.1 // indirect k8s.io/component-base v0.34.0-rc.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect diff --git a/pkg/metrics/generate_integration_test.go b/pkg/metrics/generate_integration_test.go new file mode 100644 index 000000000..6d452d9a0 --- /dev/null +++ b/pkg/metrics/generate_integration_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2023 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package metrics + +import ( + "bytes" + "fmt" + "io" + "os" + "path" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "sigs.k8s.io/controller-tools/pkg/genall" + "sigs.k8s.io/controller-tools/pkg/loader" + "sigs.k8s.io/controller-tools/pkg/markers" + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +func Test_Generate(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Error(err) + } + + optionsRegistry := &markers.Registry{} + + metricGenerator := Generator{} + if err := metricGenerator.RegisterMarkers(optionsRegistry); err != nil { + t.Error(err) + } + + out := &outputRule{ + buf: &bytes.Buffer{}, + } + + // Load the passed packages as roots. + roots, err := loader.LoadRoots(path.Join(cwd, "testdata", "...")) + if err != nil { + t.Errorf("loading packages %v", err) + } + + gen := Generator{} + + generationContext := &genall.GenerationContext{ + Collector: &markers.Collector{Registry: optionsRegistry}, + Roots: roots, + Checker: &loader.TypeChecker{}, + OutputRule: out, + } + + t.Log("Trying to generate a custom resource configuration from the loaded packages") + + if err := gen.Generate(generationContext); err != nil { + t.Error(err) + } + + output := strings.Split(out.buf.String(), "\n---\n") + + header := fmt.Sprintf(headerText, "(devel)", config.KubeStateMetricsVersion) + + if len(output) != 3 { + t.Error("Expected two output files, metrics configuration followed by rbac.") + return + } + + generatedData := map[string]string{ + "metrics.yaml": header + "---\n" + output[1], + "rbac.yaml": "---\n" + output[2], + } + + t.Log("Comparing output to testdata to check for regressions") + + for _, golden := range []string{"metrics.yaml", "rbac.yaml"} { + // generatedRaw := strings.TrimSpace(output[i]) + + expectedRaw, err := os.ReadFile(path.Clean(path.Join(cwd, "testdata", golden))) + if err != nil { + t.Error(err) + return + } + + // Remove leading `---` and trim newlines + generated := strings.TrimSpace(strings.TrimPrefix(generatedData[golden], "---")) + expected := strings.TrimSpace(strings.TrimPrefix(string(expectedRaw), "---")) + + diff := cmp.Diff(expected, generated) + if diff != "" { + t.Log("generated:") + t.Log(generated) + t.Log("diff:") + t.Log(diff) + t.Logf("Expected output to match file `testdata/%s` but it does not.", golden) + t.Logf("If the change is intended, use `go generate ./pkg/metrics/testdata` to regenerate the `testdata/%s` file.", golden) + t.Errorf("Detected a diff between the output of the integration test and the file `testdata/%s`.", golden) + return + } + } +} + +type outputRule struct { + buf *bytes.Buffer +} + +func (o *outputRule) Open(_ *loader.Package, _ string) (io.WriteCloser, error) { + return nopCloser{o.buf}, nil +} + +type nopCloser struct { + io.Writer +} + +func (n nopCloser) Close() error { + return nil +} diff --git a/pkg/metrics/generator.go b/pkg/metrics/generator.go new file mode 100644 index 000000000..13c403674 --- /dev/null +++ b/pkg/metrics/generator.go @@ -0,0 +1,183 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package metrics contain libraries for generating custom resource metrics configurations +// for kube-state-metrics from metrics markers in Go source files. +package metrics + +import ( + "fmt" + "sort" + "strings" + + "github.com/gobuffalo/flect" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/controller-tools/pkg/crd" + "sigs.k8s.io/controller-tools/pkg/genall" + "sigs.k8s.io/controller-tools/pkg/loader" + ctrlmarkers "sigs.k8s.io/controller-tools/pkg/markers" + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" + "sigs.k8s.io/controller-tools/pkg/metrics/markers" + "sigs.k8s.io/controller-tools/pkg/rbac" + "sigs.k8s.io/controller-tools/pkg/version" +) + +// Generator generates kube-state-metrics custom resource configuration files. +type Generator struct{} + +var _ genall.Generator = &Generator{} +var _ genall.NeedsTypeChecking = &Generator{} + +// RegisterMarkers registers all markers needed by this Generator +// into the given registry. +func (g Generator) RegisterMarkers(into *ctrlmarkers.Registry) error { + for _, m := range markers.MarkerDefinitions { + if err := m.Register(into); err != nil { + return err + } + } + + return nil +} + +const headerText = `# Generated by controller-gen version %s +# Generated based on types for kube-state-metrics %s +` + +// Generate generates artifacts produced by this marker. +// It's called after RegisterMarkers has been called. +func (g Generator) Generate(ctx *genall.GenerationContext) error { + // Create the parser which is specific to the metric generator. + parser := newParser( + &crd.Parser{ + Collector: ctx.Collector, + Checker: ctx.Checker, + }, + ) + + // Loop over all passed packages. + for _, pkg := range ctx.Roots { + // skip packages which don't import metav1 because they can't define a CRD without meta v1. + metav1 := pkg.Imports()["k8s.io/apimachinery/pkg/apis/meta/v1"] + if metav1 == nil { + continue + } + + // parse the given package to feed crd.FindKubeKinds with Kubernetes Objects. + parser.NeedPackage(pkg) + + kubeKinds := crd.FindKubeKinds(parser.Parser, metav1) + if len(kubeKinds) == 0 { + // no objects in the roots + return nil + } + + // Create metrics for all Custom Resources in this package. + // This creates the customresourcestate.Resource object which contains all metric + // definitions for the Custom Resource, if it is part of the package. + for _, gv := range kubeKinds { + if err := parser.NeedResourceFor(pkg, gv); err != nil { + return err + } + } + } + + // Initialize empty customresourcestate configuration file and fill it with the + // customresourcestate.Resource objects from the parser. + metrics := config.Metrics{ + Spec: config.MetricsSpec{ + Resources: []config.Resource{}, + }, + } + + rules := []*rbac.Rule{} + + for _, resource := range parser.CustomResourceStates { + if resource == nil { + continue + } + if len(resource.Metrics) > 0 { + // Sort the metrics to get a deterministic output. + sort.Slice(resource.Metrics, func(i, j int) bool { + return resource.Metrics[i].Name < resource.Metrics[j].Name + }) + + metrics.Spec.Resources = append(metrics.Spec.Resources, *resource) + + rules = append(rules, &rbac.Rule{ + Groups: []string{resource.GroupVersionKind.Group}, + Resources: []string{strings.ToLower(flect.Pluralize(resource.GroupVersionKind.Kind))}, + Verbs: []string{"get", "list", "watch"}, + }) + } + } + + // Sort the resources by GVK to get a deterministic output. + sort.Slice(metrics.Spec.Resources, func(i, j int) bool { + a := metrics.Spec.Resources[i].GroupVersionKind.String() + b := metrics.Spec.Resources[j].GroupVersionKind.String() + return a < b + }) + + header := fmt.Sprintf(headerText, version.Version(), config.KubeStateMetricsVersion) + + // Write the rendered yaml to the context which will result in stdout. + virtualFilePath := "metrics.yaml" + if err := ctx.WriteYAML(virtualFilePath, header, []interface{}{metrics}, genall.WithTransform(addCustomResourceStateKind)); err != nil { + return fmt.Errorf("WriteYAML to %s: %w", virtualFilePath, err) + } + + clusterRole := rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: rbacv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "manager-metrics-role", + Labels: map[string]string{ + "kube-state-metrics/aggregate-to-manager": "true", + }, + }, + Rules: rbac.NormalizeRules(rules), + } + + virtualFilePath = "rbac.yaml" + if err := ctx.WriteYAML(virtualFilePath, "", []interface{}{clusterRole}, genall.WithTransform(genall.TransformRemoveCreationTimestamp)); err != nil { + return fmt.Errorf("WriteYAML to %s: %w", virtualFilePath, err) + } + + return nil +} + +// CheckFilter indicates the loader.NodeFilter (if any) that should be used +// to prune out unused types/packages when type-checking (nodes for which +// the filter returns true are considered "interesting"). This filter acts +// as a baseline -- all types the pass through this filter will be checked, +// but more than that may also be checked due to other generators' filters. +func (Generator) CheckFilter() loader.NodeFilter { + // Re-use controller-tools filter to filter out unrelated nodes that aren't used + // in CRD generation, like interfaces and struct fields without JSON tag. + return crd.Generator{}.CheckFilter() +} + +// addCustomResourceStateKind adds the correct kind because we don't have a correct +// kubernetes-style object as configuration definition. +func addCustomResourceStateKind(obj map[string]interface{}) error { + obj["kind"] = "CustomResourceStateMetrics" + return nil +} diff --git a/pkg/metrics/internal/config/config.go b/pkg/metrics/internal/config/config.go new file mode 100644 index 000000000..a52f9b2eb --- /dev/null +++ b/pkg/metrics/internal/config/config.go @@ -0,0 +1,112 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" +) + +// Metrics is the top level configuration object. +type Metrics struct { + Spec MetricsSpec `yaml:"spec" json:"spec"` +} + +// MetricsSpec is the configuration describing the custom resource state metrics to generate. +type MetricsSpec struct { + // Resources is the list of custom resources to be monitored. A resource with the same GroupVersionKind may appear + // multiple times (e.g., to customize the namespace or subsystem,) but will incur additional overhead. + Resources []Resource `yaml:"resources" json:"resources"` +} + +// Resource configures a custom resource for metric generation. +type Resource struct { + // MetricNamePrefix defines a prefix for all metrics of the resource. + // If set to "", no prefix will be added. + // Example: If set to "foo", MetricNamePrefix will be "foo_". + MetricNamePrefix *string `yaml:"metricNamePrefix" json:"metricNamePrefix"` + + // GroupVersionKind of the custom resource to be monitored. + GroupVersionKind GroupVersionKind `yaml:"groupVersionKind" json:"groupVersionKind"` + + // Labels are added to all metrics. If the same key is used in a metric, the value from the metric will overwrite the value here. + Labels `yaml:",inline" json:",inline"` + + // Metrics are the custom resource fields to be collected. + Metrics []Generator `yaml:"metrics" json:"metrics"` + // ErrorLogV defines the verbosity threshold for errors logged for this resource. + ErrorLogV int32 `yaml:"errorLogV" json:"errorLogV"` + + // ResourcePlural sets the plural name of the resource. Defaults to the plural version of the Kind according to flect.Pluralize. + ResourcePlural string `yaml:"resourcePlural" json:"resourcePlural"` +} + +// GroupVersionKind is the Kubernetes group, version, and kind of a resource. +type GroupVersionKind struct { + Group string `yaml:"group" json:"group"` + Version string `yaml:"version" json:"version"` + Kind string `yaml:"kind" json:"kind"` +} + +func (gvk GroupVersionKind) String() string { + return fmt.Sprintf("%s_%s_%s", gvk.Group, gvk.Version, gvk.Kind) +} + +// Labels is common configuration of labels to add to metrics. +type Labels struct { + // CommonLabels are added to all metrics. + CommonLabels map[string]string `yaml:"commonLabels,omitempty" json:"commonLabels,omitempty"` + // LabelsFromPath adds additional labels where the value is taken from a field in the resource. + LabelsFromPath map[string][]string `yaml:"labelsFromPath,omitempty" json:"labelsFromPath,omitempty"` +} + +// Generator describes a unique metric name. +type Generator struct { + // Name of the metric. Subject to prefixing based on the configuration of the Resource. + Name string `yaml:"name" json:"name"` + // Help text for the metric. + Help string `yaml:"help" json:"help"` + // Each targets a value or values from the resource. + Each Metric `yaml:"each" json:"each"` + + // Labels are added to all metrics. Labels from Each will overwrite these if using the same key. + Labels `yaml:",inline" json:",inline"` // json will inline because it is already tagged + // ErrorLogV defines the verbosity threshold for errors logged for this metric. Must be non-zero to override the resource setting. + ErrorLogV int32 `yaml:"errorLogV,omitempty" json:"errorLogV,omitempty"` +} + +// Metric defines a metric to expose. +// +union +type Metric struct { + // Type defines the type of the metric. + // +unionDiscriminator + Type MetricType `yaml:"type" json:"type"` + + // Gauge defines a gauge metric. + // +optional + Gauge *MetricGauge `yaml:"gauge,omitempty" json:"gauge,omitempty"` + // StateSet defines a state set metric. + // +optional + StateSet *MetricStateSet `yaml:"stateSet,omitempty" json:"stateSet,omitempty"` + // Info defines an info metric. + // +optional + Info *MetricInfo `yaml:"info,omitempty" json:"info,omitempty"` +} + +// ConfigDecoder is for use with FromConfig. +type ConfigDecoder interface { + Decode(v interface{}) (err error) +} diff --git a/pkg/metrics/internal/config/config_metrics_types.go b/pkg/metrics/internal/config/config_metrics_types.go new file mode 100644 index 000000000..c57c8b0e9 --- /dev/null +++ b/pkg/metrics/internal/config/config_metrics_types.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// MetricMeta are variables which may used for any metric type. +type MetricMeta struct { + // LabelsFromPath adds additional labels where the value of the label is taken from a field under Path. + LabelsFromPath map[string][]string `yaml:"labelsFromPath,omitempty" json:"labelsFromPath,omitempty"` + // Path is the path to to generate metric(s) for. + Path []string `yaml:"path" json:"path"` +} + +// MetricGauge targets a Path that may be a single value, array, or object. Arrays and objects will generate a metric per element. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge +type MetricGauge struct { + MetricMeta `yaml:",inline" json:",inline"` + + // ValueFrom is the path to a numeric field under Path that will be the metric value. + ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` + // LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key. + LabelFromKey string `yaml:"labelFromKey,omitempty" json:"labelFromKey,omitempty"` + // NilIsZero indicates that if a value is nil it will be treated as zero value. + NilIsZero bool `yaml:"nilIsZero" json:"nilIsZero"` +} + +// MetricInfo is a metric which is used to expose textual information. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info +type MetricInfo struct { + MetricMeta `yaml:",inline" json:",inline"` + // LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key. + LabelFromKey string `yaml:"labelFromKey,omitempty" json:"labelFromKey,omitempty"` +} + +// MetricStateSet is a metric which represent a series of related boolean values, also called a bitset. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset +type MetricStateSet struct { + MetricMeta `yaml:",inline" json:",inline"` + + // List is the list of values to expose a value for. + List []string `yaml:"list" json:"list"` + // LabelName is the key of the label which is used for each entry in List to expose the value. + LabelName string `yaml:"labelName" json:"labelName"` + // ValueFrom is the subpath to compare the list to. + ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` +} diff --git a/pkg/metrics/internal/config/doc.go b/pkg/metrics/internal/config/doc.go new file mode 100644 index 000000000..6f315ff2c --- /dev/null +++ b/pkg/metrics/internal/config/doc.go @@ -0,0 +1,43 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// config contains a copy of the types from k8s.io/kube-state-metrics/pkg/customresourcestate. +// The following modifications got applied: +// For `config.go`: +// * Rename the package to `config`. +// * Drop `const customResourceState`. +// * Drop all functions, only preserve structs. +// * Use `int32` instead of `klog.Level`. +// * Use `MetricType` instead of `metric.Type` +// * Add `omitempty` to: +// - `Labels.CommonLabels` +// - `Labels.LabelsFromPath` +// - `Generator.ErrorLogV` +// - `Metric.Gauge` +// - `Metric.StateSet` +// - `Metric.Info` +// +// For `config_metrics_types.go`: +// * Rename the package to `config`. +// * Add `omitempty` to: +// - `MetricMeta.LabelsFromPath +// - `MetricGauge.LabelFromkey` +// - `MetricInfo.LabelFromkey` +package config + +// KubeStateMetricsVersion defines which version of kube-state-metrics these types +// are based on and the output file should be compatible to. +const KubeStateMetricsVersion = "v2.13.0" diff --git a/pkg/metrics/internal/config/metric_types.go b/pkg/metrics/internal/config/metric_types.go new file mode 100644 index 000000000..59721b41d --- /dev/null +++ b/pkg/metrics/internal/config/metric_types.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// MetricType is the type of a metric. +type MetricType string + +// Supported metric types. +const ( + MetricTypeGauge MetricType = "Gauge" + MetricTypeStateSet MetricType = "StateSet" + MetricTypeInfo MetricType = "Info" +) diff --git a/pkg/metrics/markers/gvk.go b/pkg/metrics/markers/gvk.go new file mode 100644 index 000000000..f36d1636c --- /dev/null +++ b/pkg/metrics/markers/gvk.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +const ( + // GVKMarkerName is the marker for a GVK. Without a set GVKMarkerName the + // generator will not generate any configuration for this GVK. + GVKMarkerName = "Metrics:gvk" +) + +func init() { + MarkerDefinitions = append( + MarkerDefinitions, + must(markers.MakeDefinition(GVKMarkerName, markers.DescribesType, gvkMarker{})). + help(gvkMarker{}.Help()), + ) +} + +// +controllertools:marker:generateHelp:category=Metrics + +// gvkMarker enables the creation of a custom resource configuration entry and uses the given prefix for the metrics if configured. +type gvkMarker struct { + // NamePrefix specifies the prefix for all metrics of this resource. + // Note: This field directly maps to the metricNamePrefix field in the resource's custom resource configuration. + NamePrefix string `marker:"namePrefix,optional"` +} + +var _ ResourceMarker = gvkMarker{} + +func (n gvkMarker) ApplyToResource(resource *config.Resource) error { + if n.NamePrefix != "" { + resource.MetricNamePrefix = &n.NamePrefix + } + return nil +} diff --git a/pkg/metrics/markers/helper.go b/pkg/metrics/markers/helper.go new file mode 100644 index 000000000..0832426af --- /dev/null +++ b/pkg/metrics/markers/helper.go @@ -0,0 +1,120 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "fmt" + + "k8s.io/client-go/util/jsonpath" + ctrlmarkers "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +type markerDefinitionWithHelp struct { + *ctrlmarkers.Definition + Help *ctrlmarkers.DefinitionHelp +} + +func must(def *ctrlmarkers.Definition, err error) *markerDefinitionWithHelp { + return &markerDefinitionWithHelp{ + Definition: ctrlmarkers.Must(def, err), + } +} + +func (d *markerDefinitionWithHelp) help(help *ctrlmarkers.DefinitionHelp) *markerDefinitionWithHelp { + d.Help = help + return d +} + +func (d *markerDefinitionWithHelp) Register(reg *ctrlmarkers.Registry) error { + if err := reg.Register(d.Definition); err != nil { + return err + } + if d.Help != nil { + reg.AddHelp(d.Definition, d.Help) + } + return nil +} + +// jsonPath is a simple JSON path, i.e. without array notation. +type jsonPath string + +// Parse is implemented to overwrite how json.Marshal and json.Unmarshal handles +// this type and parses the string to a string array instead. It is inspired by +// `kubectl explain` parsing the json path parameter. +// xref: https://github.com/kubernetes/kubectl/blob/release-1.28/pkg/explain/explain.go#L35 +func (j jsonPath) Parse() ([]string, error) { + ret := []string{} + + jpp, err := jsonpath.Parse("JSONPath", `{`+string(j)+`}`) + if err != nil { + return nil, fmt.Errorf("parse JSONPath: %w", err) + } + + // Because of the way the jsonpath library works, the schema of the parser is [][]NodeList + // meaning we need to get the outer node list, make sure it's only length 1, then get the inner node + // list, and only then can we look at the individual nodes themselves. + outerNodeList := jpp.Root.Nodes + if len(outerNodeList) > 1 { + return nil, fmt.Errorf("must pass in 1 jsonpath string, got %d", len(outerNodeList)) + } + + list, ok := outerNodeList[0].(*jsonpath.ListNode) + if !ok { + return nil, fmt.Errorf("unable to typecast to jsonpath.ListNode") + } + for _, n := range list.Nodes { + nf, ok := n.(*jsonpath.FieldNode) + if !ok { + return nil, fmt.Errorf("unable to typecast to jsonpath.NodeField") + } + ret = append(ret, nf.Value) + } + + return ret, nil +} + +func newMetricMeta(basePath []string, j jsonPath, jsonLabelsFromPath map[string]jsonPath) (config.MetricMeta, error) { + path := basePath + if j != "" { + valueFrom, err := j.Parse() + if err != nil { + return config.MetricMeta{}, fmt.Errorf("failed to parse JSONPath %q", j) + } + if len(valueFrom) > 0 { + path = append(path, valueFrom...) + } + } + + labelsFromPath := map[string][]string{} + for k, v := range jsonLabelsFromPath { + path := []string{} + var err error + if v != "." { + path, err = v.Parse() + if err != nil { + return config.MetricMeta{}, fmt.Errorf("failed to parse JSONPath %q", v) + } + } + labelsFromPath[k] = path + } + + return config.MetricMeta{ + Path: path, + LabelsFromPath: labelsFromPath, + }, nil +} diff --git a/pkg/metrics/markers/helper_test.go b/pkg/metrics/markers/helper_test.go new file mode 100644 index 000000000..bd87d19d8 --- /dev/null +++ b/pkg/metrics/markers/helper_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "reflect" + "testing" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +func Test_jsonPath_Parse(t *testing.T) { + tests := []struct { + name string + j jsonPath + want []string + wantErr bool + }{ + { + name: "empty input", + j: "", + want: []string{}, + wantErr: false, + }, + { + name: "dot input", + j: ".", + want: []string{""}, + wantErr: false, + }, + { + name: "some path input", + j: ".foo.bar", + want: []string{"foo", "bar"}, + wantErr: false, + }, + { + name: "invalid character ,", + j: ".foo,.bar", + wantErr: true, + }, + { + name: "invalid closure", + j: "{.foo}", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.j.Parse() + if (err != nil) != tt.wantErr { + t.Errorf("jsonPath.Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("jsonPath.Parse() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_newMetricMeta(t *testing.T) { + tests := []struct { + name string + basePath []string + j jsonPath + jsonLabelsFromPath map[string]jsonPath + want config.MetricMeta + }{ + { + name: "with basePath and jsonpath, without jsonLabelsFromPath", + basePath: []string{"foo"}, + j: jsonPath(".bar"), + jsonLabelsFromPath: map[string]jsonPath{}, + want: config.MetricMeta{ + Path: []string{"foo", "bar"}, + LabelsFromPath: map[string][]string{}, + }, + }, + { + name: "with basePath, jsonpath and jsonLabelsFromPath", + basePath: []string{"foo"}, + j: jsonPath(".bar"), + jsonLabelsFromPath: map[string]jsonPath{"some": ".label.from.path"}, + want: config.MetricMeta{ + Path: []string{"foo", "bar"}, + LabelsFromPath: map[string][]string{ + "some": {"label", "from", "path"}, + }, + }, + }, + { + name: "no basePath, jsonpath and jsonLabelsFromPath", + basePath: []string{}, + j: jsonPath(""), + jsonLabelsFromPath: map[string]jsonPath{}, + want: config.MetricMeta{ + Path: []string{}, + LabelsFromPath: map[string][]string{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := newMetricMeta(tt.basePath, tt.j, tt.jsonLabelsFromPath); !reflect.DeepEqual(got, tt.want) { + t.Errorf("newMetricMeta() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/metrics/markers/labelfrompath.go b/pkg/metrics/markers/labelfrompath.go new file mode 100644 index 000000000..9c5fad0ba --- /dev/null +++ b/pkg/metrics/markers/labelfrompath.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "errors" + "fmt" + + "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +const ( + labelFromPathMarkerName = "Metrics:labelFromPath" +) + +func init() { + MarkerDefinitions = append( + MarkerDefinitions, + must(markers.MakeDefinition(labelFromPathMarkerName, markers.DescribesType, labelFromPathMarker{})). + help(labelFromPathMarker{}.Help()), + must(markers.MakeDefinition(labelFromPathMarkerName, markers.DescribesField, labelFromPathMarker{})). + help(labelFromPathMarker{}.Help()), + ) +} + +// +controllertools:marker:generateHelp:category=Metrics + +// labelFromPathMarker specifies additional labels for all metrics of this field or type. +type labelFromPathMarker struct { + // Name specifies the name of the label. + Name string + // JSONPath specifies the relative path to the value for the label. + JSONPath jsonPath `marker:"JSONPath"` +} + +var _ ResourceMarker = labelFromPathMarker{} + +func (n labelFromPathMarker) ApplyToResource(resource *config.Resource) error { + if resource == nil { + return errors.New("expected resource to not be nil") + } + + jsonPathElems, err := n.JSONPath.Parse() + if err != nil { + return err + } + + if resource.LabelsFromPath == nil { + resource.LabelsFromPath = map[string][]string{} + } + + if jsonPath, labelExists := resource.LabelsFromPath[n.Name]; labelExists { + if len(jsonPathElems) != len(jsonPath) { + return fmt.Errorf("duplicate definition for label %q", n.Name) + } + for i, v := range jsonPath { + if v != jsonPathElems[i] { + return fmt.Errorf("duplicate definition for label %q", n.Name) + } + } + } + + resource.LabelsFromPath[n.Name] = jsonPathElems + return nil +} diff --git a/pkg/metrics/markers/labelfrompath_test.go b/pkg/metrics/markers/labelfrompath_test.go new file mode 100644 index 000000000..c321579c2 --- /dev/null +++ b/pkg/metrics/markers/labelfrompath_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "reflect" + "testing" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +func Test_labelFromPathMarker_ApplyToResource(t *testing.T) { + type fields struct { + Name string + JSONPath jsonPath + } + tests := []struct { + name string + fields fields + resource *config.Resource + wantResource *config.Resource + wantErr bool + }{ + { + name: "happy path", + fields: fields{ + Name: "foo", + JSONPath: ".bar", + }, + resource: &config.Resource{}, + wantResource: &config.Resource{ + Labels: config.Labels{ + LabelsFromPath: map[string][]string{ + "foo": {"bar"}, + }, + }, + }, + wantErr: false, + }, + { + name: "label already exists with same path length", + fields: fields{ + Name: "foo", + JSONPath: ".bar", + }, + resource: &config.Resource{ + Labels: config.Labels{ + LabelsFromPath: map[string][]string{ + "foo": {"other"}, + }, + }, + }, + wantResource: &config.Resource{ + Labels: config.Labels{ + LabelsFromPath: map[string][]string{ + "foo": {"other"}, + }, + }, + }, + wantErr: true, + }, + { + name: "label already exists with different path length", + fields: fields{ + Name: "foo", + JSONPath: ".bar", + }, + resource: &config.Resource{ + Labels: config.Labels{ + LabelsFromPath: map[string][]string{ + "foo": {"other", "path"}, + }, + }, + }, + wantResource: &config.Resource{ + Labels: config.Labels{ + LabelsFromPath: map[string][]string{ + "foo": {"other", "path"}, + }, + }, + }, + wantErr: true, + }, + { + name: "invalid json path", + fields: fields{ + Name: "foo", + JSONPath: "{.bar}", + }, + resource: &config.Resource{}, + wantResource: &config.Resource{}, + wantErr: true, + }, + { + name: "nil resource", + fields: fields{ + Name: "foo", + JSONPath: "{.bar}", + }, + resource: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := labelFromPathMarker{ + Name: tt.fields.Name, + JSONPath: tt.fields.JSONPath, + } + if err := n.ApplyToResource(tt.resource); (err != nil) != tt.wantErr { + t.Errorf("labelFromPathMarker.ApplyToResource() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(tt.resource, tt.wantResource) { + t.Errorf("labelFromPathMarker.ApplyToResource() = %v, want %v", tt.resource, tt.wantResource) + } + }) + } +} diff --git a/pkg/metrics/markers/markers.go b/pkg/metrics/markers/markers.go new file mode 100644 index 000000000..36ed8d1f7 --- /dev/null +++ b/pkg/metrics/markers/markers.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +var ( + // MarkerDefinitions contains all marker definitions defined by this package so + // they can get used in a generator. + MarkerDefinitions = []*markerDefinitionWithHelp{ + // GroupName is a marker copied from controller-runtime to identify the API Group. + // It needs to get added as marker so the parser will be able to read the API + // which is Group set for a package. + must(markers.MakeDefinition("groupName", markers.DescribesPackage, "")), + } +) + +// ResourceMarker is a marker that configures a custom resource. +type ResourceMarker interface { + // ApplyToCRD applies this marker to the given CRD, in the given version + // within that CRD. It's called after everything else in the CRD is populated. + ApplyToResource(resource *config.Resource) error +} + +// LocalGeneratorMarker is a marker that creates a custom resource metric generator. +type LocalGeneratorMarker interface { + // ApplyToCRD applies this marker to the given CRD, in the given version + // within that CRD. It's called after everything else in the CRD is populated. + ToGenerator(basePath ...string) (*config.Generator, error) +} diff --git a/pkg/metrics/markers/metric_gauge.go b/pkg/metrics/markers/metric_gauge.go new file mode 100644 index 000000000..e8a46420c --- /dev/null +++ b/pkg/metrics/markers/metric_gauge.go @@ -0,0 +1,102 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "fmt" + + "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +const ( + gaugeMarkerName = "Metrics:gauge" +) + +func init() { + MarkerDefinitions = append( + MarkerDefinitions, + must(markers.MakeDefinition(gaugeMarkerName, markers.DescribesField, gaugeMarker{})). + help(gaugeMarker{}.Help()), + must(markers.MakeDefinition(gaugeMarkerName, markers.DescribesType, gaugeMarker{})). + help(gaugeMarker{}.Help()), + ) +} + +// +controllertools:marker:generateHelp:category=Metric type Gauge + +// gaugeMarker defines a Gauge metric and uses the implicit path to the field joined by the provided JSONPath as path for the metric configuration. +// Gauge is a metric which targets a Path that may be a single value, array, or object. +// Arrays and objects will generate a metric per element and requre ValueFrom to be set. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge +type gaugeMarker struct { + // Keys from the Generator struct. + + // Name specifies the Name of the metric. + Name string + // MetricHelp specifies the help text for the metric. + MetricHelp string `marker:"help,optional"` + + // Keys from the MetricMeta struct. + + // LabelsFromPath specifies additional labels where the value is taken from the given JSONPath. + LabelsFromPath map[string]jsonPath `marker:"labelsFromPath,optional"` + // JSONPath specifies the relative path from this marker. + // Note: This field get's appended to the path field in the custom resource configuration. + JSONPath jsonPath `marker:"JSONPath,optional"` + + // Keys from the MetricGauge struct. + + // ValueFrom specifies the JSONPath to a numeric field that will be the metric value. + ValueFrom *jsonPath `marker:"valueFrom,optional"` + // LabelFromKey specifies a label which will be added to the metric having the object's key as value. + LabelFromKey string `marker:"labelFromKey,optional"` + // NilIsZero specifies to treat a not-existing field as zero value. + NilIsZero bool `marker:"nilIsZero,optional"` +} + +var _ LocalGeneratorMarker = &gaugeMarker{} + +func (g gaugeMarker) ToGenerator(basePath ...string) (*config.Generator, error) { + var err error + var valueFrom []string + if g.ValueFrom != nil { + valueFrom, err = g.ValueFrom.Parse() + if err != nil { + return nil, fmt.Errorf("failed to parse ValueFrom: %w", err) + } + } + + meta, err := newMetricMeta(basePath, g.JSONPath, g.LabelsFromPath) + if err != nil { + return nil, err + } + + return &config.Generator{ + Name: g.Name, + Help: g.MetricHelp, + Each: config.Metric{ + Type: config.MetricTypeGauge, + Gauge: &config.MetricGauge{ + NilIsZero: g.NilIsZero, + MetricMeta: meta, + LabelFromKey: g.LabelFromKey, + ValueFrom: valueFrom, + }, + }, + }, nil +} diff --git a/pkg/metrics/markers/metric_gauge_test.go b/pkg/metrics/markers/metric_gauge_test.go new file mode 100644 index 000000000..083308277 --- /dev/null +++ b/pkg/metrics/markers/metric_gauge_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "reflect" + "testing" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +func Test_gaugeMarker_ToGenerator(t *testing.T) { + tests := []struct { + name string + gaugeMarker gaugeMarker + basePath []string + want *config.Generator + }{ + { + name: "Happy path", + gaugeMarker: gaugeMarker{ + ValueFrom: jsonPathPointer(".foo"), + }, + basePath: []string{}, + want: &config.Generator{ + Each: config.Metric{ + Type: config.MetricTypeGauge, + Gauge: &config.MetricGauge{ + MetricMeta: config.MetricMeta{ + LabelsFromPath: map[string][]string{}, + Path: []string{}, + }, + ValueFrom: []string{"foo"}, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := tt.gaugeMarker.ToGenerator(tt.basePath...); !reflect.DeepEqual(got, tt.want) { + t.Errorf("gaugeMarker.ToGenerator() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/metrics/markers/metric_info.go b/pkg/metrics/markers/metric_info.go new file mode 100644 index 000000000..83562856f --- /dev/null +++ b/pkg/metrics/markers/metric_info.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +const ( + infoMarkerName = "Metrics:info" +) + +func init() { + MarkerDefinitions = append( + MarkerDefinitions, + must(markers.MakeDefinition(infoMarkerName, markers.DescribesField, infoMarker{})). + help(infoMarker{}.Help()), + must(markers.MakeDefinition(infoMarkerName, markers.DescribesType, infoMarker{})). + help(infoMarker{}.Help()), + ) +} + +// +controllertools:marker:generateHelp:category=Metric type Info + +// infoMarker defines a Info metric and uses the implicit path to the field as path for the metric configuration. +// Info is a metric which is used to expose textual information. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info +type infoMarker struct { + // Keys from the Generator struct. + + // Name specifies the Name of the metric. + Name string + // MetricHelp specifies the help text for the metric. + MetricHelp string `marker:"help,optional"` + + // Keys from the MetricMeta struct. + + // LabelsFromPath specifies additional labels where the value is taken from the given JSONPath. + LabelsFromPath map[string]jsonPath `marker:"labelsFromPath,optional"` + // JSONPath specifies the relative path from this marker. + // Note: This field get's appended to the path field in the custom resource configuration. + JSONPath jsonPath `marker:"JSONPath,optional"` + + // Keys from the MetricInfo struct. + + // LabelFromKey specifies a label which will be added to the metric having the object's key as value. + LabelFromKey string `marker:"labelFromKey,optional"` +} + +var _ LocalGeneratorMarker = &infoMarker{} + +func (i infoMarker) ToGenerator(basePath ...string) (*config.Generator, error) { + meta, err := newMetricMeta(basePath, i.JSONPath, i.LabelsFromPath) + if err != nil { + return nil, err + } + + return &config.Generator{ + Name: i.Name, + Help: i.MetricHelp, + Each: config.Metric{ + Type: config.MetricTypeInfo, + Info: &config.MetricInfo{ + MetricMeta: meta, + LabelFromKey: i.LabelFromKey, + }, + }, + }, nil +} diff --git a/pkg/metrics/markers/metric_info_test.go b/pkg/metrics/markers/metric_info_test.go new file mode 100644 index 000000000..a4ff6aded --- /dev/null +++ b/pkg/metrics/markers/metric_info_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "reflect" + "testing" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +func Test_infoMarker_ToGenerator(t *testing.T) { + tests := []struct { + name string + infoMarker infoMarker + basePath []string + want *config.Generator + }{ + { + name: "Happy path", + infoMarker: infoMarker{}, + basePath: []string{}, + want: &config.Generator{ + Each: config.Metric{ + Type: config.MetricTypeInfo, + Info: &config.MetricInfo{ + MetricMeta: config.MetricMeta{ + LabelsFromPath: map[string][]string{}, + Path: []string{}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := tt.infoMarker.ToGenerator(tt.basePath...); !reflect.DeepEqual(got, tt.want) { + t.Errorf("infoMarker.ToGenerator() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/metrics/markers/metric_stateset.go b/pkg/metrics/markers/metric_stateset.go new file mode 100644 index 000000000..3f006c805 --- /dev/null +++ b/pkg/metrics/markers/metric_stateset.go @@ -0,0 +1,99 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "fmt" + + "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +const ( + stateSetMarkerName = "Metrics:stateset" +) + +func init() { + MarkerDefinitions = append( + MarkerDefinitions, + must(markers.MakeDefinition(stateSetMarkerName, markers.DescribesField, stateSetMarker{})). + help(stateSetMarker{}.Help()), + must(markers.MakeDefinition(stateSetMarkerName, markers.DescribesType, stateSetMarker{})). + help(stateSetMarker{}.Help()), + ) +} + +// +controllertools:marker:generateHelp:category=Metric type StateSet + +// stateSetMarker defines a StateSet metric and uses the implicit path to the field as path for the metric configuration. +// A StateSet is a metric which represent a series of related boolean values, also called a bitset. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset +type stateSetMarker struct { + // Keys from the Generator struct. + + // Name specifies the Name of the metric. + Name string + // MetricHelp specifies the help text for the metric. + MetricHelp string `marker:"help,optional"` + + // Keys from the MetricMeta struct. + + // LabelsFromPath specifies additional labels where the value is taken from the given JSONPath. + LabelsFromPath map[string]jsonPath `marker:"labelsFromPath,optional"` + + // Keys from the MetricStateSet struct. + + // List specifies a list of values to compare the given JSONPath against. + List []string `marker:"list"` + // LabelName specifies the key of the label which is used for each entry in List to expose the value. + LabelName string `marker:"labelName,optional"` + // JSONPath specifies the path to the field which gets used as value to compare against the list for equality. + // Note: This field directly maps to the valueFrom field in the custom resource configuration. + JSONPath *jsonPath `marker:"JSONPath,optional"` +} + +var _ LocalGeneratorMarker = &stateSetMarker{} + +func (s stateSetMarker) ToGenerator(basePath ...string) (*config.Generator, error) { + var valueFrom []string + var err error + if s.JSONPath != nil { + valueFrom, err = s.JSONPath.Parse() + if err != nil { + return nil, fmt.Errorf("failed to parse JSONPath: %w", err) + } + } + + meta, err := newMetricMeta(basePath, "", s.LabelsFromPath) + if err != nil { + return nil, err + } + + return &config.Generator{ + Name: s.Name, + Help: s.MetricHelp, + Each: config.Metric{ + Type: config.MetricTypeStateSet, + StateSet: &config.MetricStateSet{ + MetricMeta: meta, + List: s.List, + LabelName: s.LabelName, + ValueFrom: valueFrom, + }, + }, + }, nil +} diff --git a/pkg/metrics/markers/metric_stateset_test.go b/pkg/metrics/markers/metric_stateset_test.go new file mode 100644 index 000000000..e293660b0 --- /dev/null +++ b/pkg/metrics/markers/metric_stateset_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package markers + +import ( + "reflect" + "testing" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" +) + +func Test_stateSetMarker_ToGenerator(t *testing.T) { + tests := []struct { + name string + stateSetMarker stateSetMarker + basePath []string + want *config.Generator + }{ + { + name: "Happy path", + stateSetMarker: stateSetMarker{ + JSONPath: jsonPathPointer(".foo"), + }, + basePath: []string{}, + want: &config.Generator{ + Each: config.Metric{ + Type: config.MetricTypeStateSet, + StateSet: &config.MetricStateSet{ + MetricMeta: config.MetricMeta{ + LabelsFromPath: map[string][]string{}, + Path: []string{}, + }, + ValueFrom: []string{"foo"}, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := tt.stateSetMarker.ToGenerator(tt.basePath...); !reflect.DeepEqual(got, tt.want) { + t.Errorf("stateSetMarker.ToGenerator() = %v, want %v", got, tt.want) + } + }) + } +} + +func jsonPathPointer(s string) *jsonPath { + j := jsonPath(s) + return &j +} diff --git a/pkg/metrics/markers/zz_generated.markerhelp.go b/pkg/metrics/markers/zz_generated.markerhelp.go new file mode 100644 index 000000000..4a5ea7309 --- /dev/null +++ b/pkg/metrics/markers/zz_generated.markerhelp.go @@ -0,0 +1,169 @@ +//go:build !ignore_autogenerated + +/* +Copyright2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by helpgen. DO NOT EDIT. + +package markers + +import ( + "sigs.k8s.io/controller-tools/pkg/markers" +) + +func (gaugeMarker) Help() *markers.DefinitionHelp { + return &markers.DefinitionHelp{ + Category: "Metric type Gauge", + DetailedHelp: markers.DetailedHelp{ + Summary: "defines a Gauge metric and uses the implicit path to the field joined by the provided JSONPath as path for the metric configuration.", + Details: "Gauge is a metric which targets a Path that may be a single value, array, or object.\nArrays and objects will generate a metric per element and requre ValueFrom to be set.\nRef: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge", + }, + FieldHelp: map[string]markers.DetailedHelp{ + "Name": { + Summary: "specifies the Name of the metric.", + Details: "", + }, + "MetricHelp": { + Summary: "specifies the help text for the metric.", + Details: "", + }, + "LabelsFromPath": { + Summary: "specifies additional labels where the value is taken from the given JSONPath.", + Details: "", + }, + "JSONPath": { + Summary: "specifies the relative path from this marker.", + Details: "Note: This field get's appended to the path field in the custom resource configuration.", + }, + "ValueFrom": { + Summary: "specifies the JSONPath to a numeric field that will be the metric value.", + Details: "", + }, + "LabelFromKey": { + Summary: "specifies a label which will be added to the metric having the object's key as value.", + Details: "", + }, + "NilIsZero": { + Summary: "specifies to treat a not-existing field as zero value.", + Details: "", + }, + }, + } +} + +func (gvkMarker) Help() *markers.DefinitionHelp { + return &markers.DefinitionHelp{ + Category: "Metrics", + DetailedHelp: markers.DetailedHelp{ + Summary: "enables the creation of a custom resource configuration entry and uses the given prefix for the metrics if configured.", + Details: "", + }, + FieldHelp: map[string]markers.DetailedHelp{ + "NamePrefix": { + Summary: "specifies the prefix for all metrics of this resource.", + Details: "Note: This field directly maps to the metricNamePrefix field in the resource's custom resource configuration.", + }, + }, + } +} + +func (infoMarker) Help() *markers.DefinitionHelp { + return &markers.DefinitionHelp{ + Category: "Metric type Info", + DetailedHelp: markers.DetailedHelp{ + Summary: "defines a Info metric and uses the implicit path to the field as path for the metric configuration.", + Details: "Info is a metric which is used to expose textual information.\nRef: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info", + }, + FieldHelp: map[string]markers.DetailedHelp{ + "Name": { + Summary: "specifies the Name of the metric.", + Details: "", + }, + "MetricHelp": { + Summary: "specifies the help text for the metric.", + Details: "", + }, + "LabelsFromPath": { + Summary: "specifies additional labels where the value is taken from the given JSONPath.", + Details: "", + }, + "JSONPath": { + Summary: "specifies the relative path from this marker.", + Details: "Note: This field get's appended to the path field in the custom resource configuration.", + }, + "LabelFromKey": { + Summary: "specifies a label which will be added to the metric having the object's key as value.", + Details: "", + }, + }, + } +} + +func (labelFromPathMarker) Help() *markers.DefinitionHelp { + return &markers.DefinitionHelp{ + Category: "Metrics", + DetailedHelp: markers.DetailedHelp{ + Summary: "specifies additional labels for all metrics of this field or type.", + Details: "", + }, + FieldHelp: map[string]markers.DetailedHelp{ + "Name": { + Summary: "specifies the name of the label.", + Details: "", + }, + "JSONPath": { + Summary: "specifies the relative path to the value for the label.", + Details: "", + }, + }, + } +} + +func (stateSetMarker) Help() *markers.DefinitionHelp { + return &markers.DefinitionHelp{ + Category: "Metric type StateSet", + DetailedHelp: markers.DetailedHelp{ + Summary: "defines a StateSet metric and uses the implicit path to the field as path for the metric configuration.", + Details: "A StateSet is a metric which represent a series of related boolean values, also called a bitset.\nRef: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset", + }, + FieldHelp: map[string]markers.DetailedHelp{ + "Name": { + Summary: "specifies the Name of the metric.", + Details: "", + }, + "MetricHelp": { + Summary: "specifies the help text for the metric.", + Details: "", + }, + "LabelsFromPath": { + Summary: "specifies additional labels where the value is taken from the given JSONPath.", + Details: "", + }, + "List": { + Summary: "specifies a list of values to compare the given JSONPath against.", + Details: "", + }, + "LabelName": { + Summary: "specifies the key of the label which is used for each entry in List to expose the value.", + Details: "", + }, + "JSONPath": { + Summary: "specifies the path to the field which gets used as value to compare against the list for equality.", + Details: "Note: This field directly maps to the valueFrom field in the custom resource configuration.", + }, + }, + } +} diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go new file mode 100644 index 000000000..cea306be7 --- /dev/null +++ b/pkg/metrics/parser.go @@ -0,0 +1,275 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package metrics + +import ( + "fmt" + "go/ast" + "go/types" + "strings" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-tools/pkg/crd" + "sigs.k8s.io/controller-tools/pkg/loader" + ctrlmarkers "sigs.k8s.io/controller-tools/pkg/markers" + + "sigs.k8s.io/controller-tools/pkg/metrics/internal/config" + "sigs.k8s.io/controller-tools/pkg/metrics/markers" +) + +type parser struct { + *crd.Parser + + CustomResourceStates map[crd.TypeIdent]*config.Resource +} + +func newParser(p *crd.Parser) *parser { + return &parser{ + Parser: p, + CustomResourceStates: make(map[crd.TypeIdent]*config.Resource), + } +} + +// NeedResourceFor creates the customresourcestate.Resource object for the given +// GroupKind located at the package identified by packageID. +func (p *parser) NeedResourceFor(pkg *loader.Package, groupKind schema.GroupKind) error { + typeIdent := crd.TypeIdent{Package: pkg, Name: groupKind.Kind} + // Skip if type was already processed. + if _, exists := p.CustomResourceStates[typeIdent]; exists { + return nil + } + + // Already mark the cacheID so the next time it enters NeedResourceFor it skips early. + p.CustomResourceStates[typeIdent] = nil + + // Build the type identifier for the custom resource. + typeInfo := p.Types[typeIdent] + // typeInfo is nil if this GroupKind is not part of this package. In that case + // we have nothing to process. + if typeInfo == nil { + return nil + } + + // Skip if gvk marker is not set. This marker is the opt-in for creating metrics + // for a custom resource. + if m := typeInfo.Markers.Get(markers.GVKMarkerName); m == nil { + return nil + } + + metrics, err := p.NeedMetricsGeneratorFor(typeIdent) + if err != nil { + return err + } + + // Initialize the Resource object. + resource := config.Resource{ + GroupVersionKind: config.GroupVersionKind{ + Group: groupKind.Group, + Kind: groupKind.Kind, + Version: p.GroupVersions[pkg].Version, + }, + // Create the metrics generators for the custom resource. + Metrics: metrics, + } + + // Iterate through all markers and run the ApplyToResource function of the ResourceMarkers. + for _, markerVals := range typeInfo.Markers { + for _, val := range markerVals { + if resourceMarker, isResourceMarker := val.(markers.ResourceMarker); isResourceMarker { + if err := resourceMarker.ApplyToResource(&resource); err != nil { + pkg.AddError(loader.ErrFromNode(err /* an okay guess */, typeInfo.RawSpec)) + } + } + } + } + + p.CustomResourceStates[typeIdent] = &resource + return nil +} + +type generatorRequester interface { + NeedMetricsGeneratorFor(typ crd.TypeIdent) ([]config.Generator, error) +} + +// generatorContext stores and provides information across a hierarchy of metric generators generation. +type generatorContext struct { + pkg *loader.Package + generatorRequester generatorRequester + + PackageMarkers ctrlmarkers.MarkerValues +} + +func newGeneratorContext(pkg *loader.Package, req generatorRequester) *generatorContext { + pkg.NeedTypesInfo() + return &generatorContext{ + pkg: pkg, + generatorRequester: req, + } +} + +func generatorsFromMarkers(m ctrlmarkers.MarkerValues, basePath ...string) ([]config.Generator, error) { + generators := []config.Generator{} + + for _, markerVals := range m { + for _, val := range markerVals { + if generatorMarker, isGeneratorMarker := val.(markers.LocalGeneratorMarker); isGeneratorMarker { + g, err := generatorMarker.ToGenerator(basePath...) + if err != nil { + return nil, err + } + if g != nil { + generators = append(generators, *g) + } + } + } + } + + return generators, nil +} + +// NeedMetricsGeneratorFor creates the customresourcestate.Generator object for a +// Custom Resource. +func (p *parser) NeedMetricsGeneratorFor(typ crd.TypeIdent) ([]config.Generator, error) { + info, gotInfo := p.Types[typ] + if !gotInfo { + return nil, fmt.Errorf("type info for %v does not exist", typ) + } + + // Add metric allGenerators defined by markers at the type. + allGenerators, err := generatorsFromMarkers(info.Markers) + if err != nil { + return nil, err + } + + // Traverse fields of the object and process markers. + // Note: Partially inspired by controller-tools. + // xref: https://github.com/kubernetes-sigs/controller-tools/blob/d89d6ae3df218a85f7cd9e477157cace704b37d1/pkg/crd/schema.go#L350 + for _, f := range info.Fields { + // Only fields with the `json:"..."` tag are relevant. Others are not part of the Custom Resource. + jsonTag, hasTag := f.Tag.Lookup("json") + if !hasTag { + // if the field doesn't have a JSON tag, it doesn't belong in output (and shouldn't exist in a serialized type) + continue + } + jsonOpts := strings.Split(jsonTag, ",") + if len(jsonOpts) == 1 && jsonOpts[0] == "-" { + // skipped fields have the tag "-" (note that "-," means the field is named "-") + continue + } + + // Add metric markerGenerators defined by markers at the field. + markerGenerators, err := generatorsFromMarkers(f.Markers, jsonOpts[0]) + if err != nil { + return nil, err + } + allGenerators = append(allGenerators, markerGenerators...) + + // Create new generator context and recursively process the fields. + generatorCtx := newGeneratorContext(typ.Package, p) + generators, err := generatorsFor(generatorCtx, f.RawField.Type) + if err != nil { + return nil, err + } + for _, generator := range generators { + allGenerators = append(allGenerators, addPathPrefixOnGenerator(generator, jsonOpts[0])) + } + } + + return allGenerators, nil +} + +// generatorsFor creates generators for the given AST type. +// Note: Partially inspired by controller-tools. +// xref: https://github.com/kubernetes-sigs/controller-tools/blob/d89d6ae3df218a85f7cd9e477157cace704b37d1/pkg/crd/schema.go#L167-L193 +func generatorsFor(ctx *generatorContext, rawType ast.Expr) ([]config.Generator, error) { + switch expr := rawType.(type) { + case *ast.Ident: + return localNamedToGenerators(ctx, expr) + case *ast.SelectorExpr: + // Results in using transitive markers from external packages. + return generatorsFor(ctx, expr.X) + case *ast.ArrayType: + // The current configuration does not allow creating metric configurations inside arrays + return nil, nil + case *ast.MapType: + // The current configuration does not allow creating metric configurations inside maps + return nil, nil + case *ast.StarExpr: + return generatorsFor(ctx, expr.X) + case *ast.StructType: + ctx.pkg.AddError(loader.ErrFromNode(fmt.Errorf("unsupported AST kind %T", expr), rawType)) + default: + ctx.pkg.AddError(loader.ErrFromNode(fmt.Errorf("unsupported AST kind %T", expr), rawType)) + // NB(directxman12): we explicitly don't handle interfaces + return nil, nil + } + + return nil, nil +} + +// localNamedToGenerators recurses back to NeedMetricsGeneratorFor for the type to +// get generators defined at the objects in a custom resource. +func localNamedToGenerators(ctx *generatorContext, ident *ast.Ident) ([]config.Generator, error) { + typeInfo := ctx.pkg.TypesInfo.TypeOf(ident) + if typeInfo == types.Typ[types.Invalid] { + // It is expected to hit this error for types from not loaded transitive package dependencies. + // This leads to ignoring markers defined on the transitive types. Otherwise + // markers on transitive types would lead to additional metrics. + return nil, nil + } + + if _, isBasic := typeInfo.(*types.Basic); isBasic { + // There can't be markers for basic go types for this generator. + return nil, nil + } + + // NB(directxman12): if there are dot imports, this might be an external reference, + // so use typechecking info to get the actual object + typeNameInfo := typeInfo.(*types.Named).Obj() + pkg := typeNameInfo.Pkg() + pkgPath := loader.NonVendorPath(pkg.Path()) + if pkg == ctx.pkg.Types { + pkgPath = "" + } + return ctx.requestGenerator(pkgPath, typeNameInfo.Name()) +} + +// requestGenerator asks for the generator for a type in the package with the +// given import path. +func (c *generatorContext) requestGenerator(pkgPath, typeName string) ([]config.Generator, error) { + pkg := c.pkg + if pkgPath != "" { + pkg = c.pkg.Imports()[pkgPath] + } + return c.generatorRequester.NeedMetricsGeneratorFor(crd.TypeIdent{ + Package: pkg, + Name: typeName, + }) +} + +// addPathPrefixOnGenerator prefixes the path set at the generators MetricMeta object. +func addPathPrefixOnGenerator(generator config.Generator, pathPrefix string) config.Generator { + switch generator.Each.Type { + case config.MetricTypeGauge: + generator.Each.Gauge.MetricMeta.Path = append([]string{pathPrefix}, generator.Each.Gauge.MetricMeta.Path...) + case config.MetricTypeStateSet: + generator.Each.StateSet.MetricMeta.Path = append([]string{pathPrefix}, generator.Each.StateSet.MetricMeta.Path...) + case config.MetricTypeInfo: + generator.Each.Info.MetricMeta.Path = append([]string{pathPrefix}, generator.Each.Info.MetricMeta.Path...) + } + + return generator +} diff --git a/pkg/metrics/testdata/README.md b/pkg/metrics/testdata/README.md new file mode 100644 index 000000000..317d67c48 --- /dev/null +++ b/pkg/metrics/testdata/README.md @@ -0,0 +1,64 @@ +# Testdata for generator tests + +The files in this directory are used for testing the `kube-state-metrics generate` command and to provide an example. + +## foo-config.yaml + +This file is used in the test at [generate_integration_test.go](../generate_integration_test.go) to verify that the resulting configuration does not change during changes in the codebase. + +If there are intended changes this file needs to get regenerated to make the test succeed again. +This could be done via: + +```sh +go run ./cmd/controller-gen metrics crd \ + paths=./pkg/metrics/testdata \ + output:dir=./pkg/metrics/testdata +``` + +Or by using the go:generate marker inside [foo_types.go](foo_types.go): + +```sh +go generate ./pkg/metrics/testdata/ +``` + +## Example files: metrics.yaml, rbac.yaml and example-metrics.txt + +There is also an example CR ([example-foo.yaml](example-foo.yaml)) and resulting example metrics ([example-metrics.txt](example-metrics.txt)). + +The example metrics file got created by: + +1. Generating a CustomResourceDefinition and Kube-State-Metrics configration file: + + ```sh + go generate ./pkg/metrics/testdata/ + ``` + +2. Creating a cluster using [kind](https://kind.sigs.k8s.io/) + + ```sh + kind create cluster + ``` + +3. Applying the CRD and example CR to the cluster: + + ```sh + kubectl apply -f ./pkg/metrics/testdata/bar.example.com_foos.yaml + kubectl apply -f ./pkg/metrics/testdata/example-foo.yaml + ``` + +4. Running kube-state-metrics with the provided configuration file: + + ```sh + docker run --net=host -ti --rm \ + -v $HOME/.kube/config:/config \ + -v $(pwd):/data \ + registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0 \ + --kubeconfig /config --custom-resource-state-only \ + --custom-resource-state-config-file /data/pkg/metrics/testdata/foo-config.yaml + ``` + +5. Querying the metrics endpoint in a second terminal: + + ```sh + curl localhost:8080/metrics > ./pkg/metrics/testdata/foo-cr-example-metrics.txt + ``` diff --git a/pkg/metrics/testdata/bar.example.com_foos.yaml b/pkg/metrics/testdata/bar.example.com_foos.yaml new file mode 100644 index 000000000..edbb2ecd0 --- /dev/null +++ b/pkg/metrics/testdata/bar.example.com_foos.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + name: foos.bar.example.com +spec: + group: bar.example.com + names: + kind: Foo + listKind: FooList + plural: foos + singular: foo + scope: Namespaced + versions: + - name: foo + schema: + openAPIV3Schema: + description: Foo is a test object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec comments SHOULD appear in the CRD spec + properties: + someString: + description: SomeString is a string. + type: string + required: + - someString + type: object + status: + description: Status comments SHOULD appear in the CRD spec + properties: + conditions: + items: + description: Condition is a test condition. + properties: + lastTransitionTime: + description: LastTransitionTime of condition. + format: date-time + type: string + status: + description: Status of condition. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true diff --git a/pkg/metrics/testdata/example-foo.yaml b/pkg/metrics/testdata/example-foo.yaml new file mode 100644 index 000000000..bcf1243ef --- /dev/null +++ b/pkg/metrics/testdata/example-foo.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: bar.example.com/foo +kind: Foo +metadata: + name: bar + ownerReferences: + - apiVersion: v1 + kind: foo + controller: true + name: foo + uid: someuid +spec: + someString: test +status: + conditions: + - lastTransitionTime: "2023-10-12T13:59:02Z" + status: "True" + type: SomeType + - lastTransitionTime: "2023-10-12T13:59:02Z" + status: "False" + type: AnotherType diff --git a/pkg/metrics/testdata/example-metrics.txt b/pkg/metrics/testdata/example-metrics.txt new file mode 100644 index 000000000..a4050cfdd --- /dev/null +++ b/pkg/metrics/testdata/example-metrics.txt @@ -0,0 +1,18 @@ +# HELP foo_created Unix creation timestamp. +# TYPE foo_created gauge +foo_created{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar"} 1.724940463e+09 +# HELP foo_owner Owner references. +# TYPE foo_owner info +foo_owner{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",owner_is_controller="true",owner_kind="foo",owner_name="foo",owner_uid="someuid"} 1 +# HELP foo_status_condition The condition of a foo. +# TYPE foo_status_condition stateset +foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="False",type="AnotherType"} 1 +foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="False",type="SomeType"} 0 +foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="True",type="AnotherType"} 0 +foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="True",type="SomeType"} 1 +foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="Unknown",type="AnotherType"} 0 +foo_status_condition{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="Unknown",type="SomeType"} 0 +# HELP foo_status_condition_last_transition_time The condition last transition time of a foo. +# TYPE foo_status_condition_last_transition_time gauge +foo_status_condition_last_transition_time{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="False",type="AnotherType"} 1.697119142e+09 +foo_status_condition_last_transition_time{customresource_group="bar.example.com",customresource_kind="Foo",customresource_version="foo",name="bar",status="True",type="SomeType"} 1.697119142e+09 diff --git a/pkg/metrics/testdata/foo_types.go b/pkg/metrics/testdata/foo_types.go new file mode 100644 index 000000000..6f95bd760 --- /dev/null +++ b/pkg/metrics/testdata/foo_types.go @@ -0,0 +1,70 @@ +/* +Copyright 2023 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Changes to this file may require to regenerate the `foo-config.yaml`. Otherwise the +// tests in ../generate_integration_test.go may fail. +// The below marker can be used to regenerate the `foo-config.yaml` file by running +// the following command: +// $ go generate ./pkg/customresourcestate/generate/generator/testdata +//go:generate sh -c "go run ../../../cmd/controller-gen crd metrics paths=./ output:dir=." + +// +groupName=bar.example.com +package foo + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// FooSpec is the spec of Foo. +type FooSpec struct { + // SomeString is a string. + SomeString string `json:"someString"` +} + +// FooStatus is the status of Foo. +type FooStatus struct { + // +Metrics:stateset:name="status_condition",help="The condition of a foo.",labelName="status",JSONPath=".status",list={"True","False","Unknown"},labelsFromPath={"type":".type"} + // +Metrics:gauge:name="status_condition_last_transition_time",help="The condition last transition time of a foo.",valueFrom=.lastTransitionTime,labelsFromPath={"type":".type","status":".status"} + Conditions []Condition `json:"conditions,omitempty"` +} + +// Foo is a test object. +// +Metrics:gvk:namePrefix="foo" +// +Metrics:labelFromPath:name="name",JSONPath=".metadata.name" +// +Metrics:gauge:name="created",JSONPath=".metadata.creationTimestamp",help="Unix creation timestamp." +// +Metrics:info:name="owner",JSONPath=".metadata.ownerReferences",help="Owner references.",labelsFromPath={owner_is_controller:".controller",owner_kind:".kind",owner_name:".name",owner_uid:".uid"} +// +Metrics:labelFromPath:name="cluster_name",JSONPath=.metadata.labels.cluster\.x-k8s\.io/cluster-name +type Foo struct { + // TypeMeta comments should NOT appear in the CRD spec + metav1.TypeMeta `json:",inline"` + // ObjectMeta comments should NOT appear in the CRD spec + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec comments SHOULD appear in the CRD spec + Spec FooSpec `json:"spec,omitempty"` + // Status comments SHOULD appear in the CRD spec + Status FooStatus `json:"status,omitempty"` +} + +// Condition is a test condition. +type Condition struct { + // Type of condition. + Type string `json:"type"` + // Status of condition. + Status string `json:"status"` + // LastTransitionTime of condition. + LastTransitionTime metav1.Time `json:"lastTransitionTime"` +} diff --git a/pkg/metrics/testdata/metrics.yaml b/pkg/metrics/testdata/metrics.yaml new file mode 100644 index 000000000..ed2b02b6f --- /dev/null +++ b/pkg/metrics/testdata/metrics.yaml @@ -0,0 +1,83 @@ +# Generated by controller-gen version (devel) +# Generated based on types for kube-state-metrics v2.13.0 +--- +kind: CustomResourceStateMetrics +spec: + resources: + - errorLogV: 0 + groupVersionKind: + group: bar.example.com + kind: Foo + version: foo + labelsFromPath: + cluster_name: + - metadata + - labels + - cluster.x-k8s.io/cluster-name + name: + - metadata + - name + metricNamePrefix: foo + metrics: + - each: + gauge: + nilIsZero: false + path: + - metadata + - creationTimestamp + valueFrom: null + type: Gauge + help: Unix creation timestamp. + name: created + - each: + info: + labelsFromPath: + owner_is_controller: + - controller + owner_kind: + - kind + owner_name: + - name + owner_uid: + - uid + path: + - metadata + - ownerReferences + type: Info + help: Owner references. + name: owner + - each: + stateSet: + labelName: status + labelsFromPath: + type: + - type + list: + - "True" + - "False" + - Unknown + path: + - status + - conditions + valueFrom: + - status + type: StateSet + help: The condition of a foo. + name: status_condition + - each: + gauge: + labelsFromPath: + status: + - status + type: + - type + nilIsZero: false + path: + - status + - conditions + valueFrom: + - lastTransitionTime + type: Gauge + help: The condition last transition time of a foo. + name: status_condition_last_transition_time + resourcePlural: "" diff --git a/pkg/metrics/testdata/rbac.yaml b/pkg/metrics/testdata/rbac.yaml new file mode 100644 index 000000000..1f37eab55 --- /dev/null +++ b/pkg/metrics/testdata/rbac.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kube-state-metrics/aggregate-to-manager: "true" + name: manager-metrics-role +rules: +- apiGroups: + - bar.example.com + resources: + - foos + verbs: + - get + - list + - watch diff --git a/pkg/rbac/parser.go b/pkg/rbac/parser.go index 6521d2658..af702738d 100644 --- a/pkg/rbac/parser.go +++ b/pkg/rbac/parser.go @@ -221,107 +221,6 @@ func GenerateRoles(ctx *genall.GenerationContext, roleName string) ([]interface{ } } - // NormalizeRules merge Rule with the same ruleKey and sort the Rules - NormalizeRules := func(rules []*Rule) []rbacv1.PolicyRule { - ruleMap := make(map[ruleKey]*Rule) - // all the Rules having the same ruleKey will be merged into the first Rule - for _, rule := range rules { - // fix the group name first, since letting people type "core" is nice - for i, name := range rule.Groups { - if name == "core" { - rule.Groups[i] = "" - } - } - - key := rule.key() - if _, ok := ruleMap[key]; !ok { - ruleMap[key] = rule - continue - } - ruleMap[key].addVerbs(rule.Verbs) - } - - // deduplicate resources - // 1. create map based on key without resources - ruleMapWithoutResources := make(map[string][]*Rule) - for _, rule := range ruleMap { - // get key without Resources - key := rule.keyWithGroupResourceNamesURLsVerbs() - ruleMapWithoutResources[key] = append(ruleMapWithoutResources[key], rule) - } - // 2. merge to ruleMap - ruleMap = make(map[ruleKey]*Rule) - for _, rules := range ruleMapWithoutResources { - rule := rules[0] - for _, mergeRule := range rules[1:] { - rule.Resources = append(rule.Resources, mergeRule.Resources...) - } - - key := rule.key() - ruleMap[key] = rule - } - - // deduplicate groups - // 1. create map based on key without group - ruleMapWithoutGroup := make(map[string][]*Rule) - for _, rule := range ruleMap { - // get key without Group - key := rule.keyWithResourcesResourceNamesURLsVerbs() - ruleMapWithoutGroup[key] = append(ruleMapWithoutGroup[key], rule) - } - // 2. merge to ruleMap - ruleMap = make(map[ruleKey]*Rule) - for _, rules := range ruleMapWithoutGroup { - rule := rules[0] - for _, mergeRule := range rules[1:] { - rule.Groups = append(rule.Groups, mergeRule.Groups...) - } - key := rule.key() - ruleMap[key] = rule - } - - // deduplicate URLs - // 1. create map based on key without URLs - ruleMapWithoutURLs := make(map[string][]*Rule) - for _, rule := range ruleMap { - // get key without Group - key := rule.keyWitGroupResourcesResourceNamesVerbs() - ruleMapWithoutURLs[key] = append(ruleMapWithoutURLs[key], rule) - } - // 2. merge to ruleMap - ruleMap = make(map[ruleKey]*Rule) - for _, rules := range ruleMapWithoutURLs { - rule := rules[0] - for _, mergeRule := range rules[1:] { - rule.URLs = append(rule.URLs, mergeRule.URLs...) - } - key := rule.key() - ruleMap[key] = rule - } - - // sort the Rules in rules according to their ruleKeys - keys := make([]ruleKey, 0, len(ruleMap)) - for key := range ruleMap { - keys = append(keys, key) - } - sort.Sort(ruleKeys(keys)) - - // Normalize rule verbs to "*" if any verb in the rule is an asterisk - for _, rule := range ruleMap { - for _, verb := range rule.Verbs { - if verb == "*" { - rule.Verbs = []string{"*"} - break - } - } - } - var policyRules []rbacv1.PolicyRule - for _, key := range keys { - policyRules = append(policyRules, ruleMap[key].ToRule()) - } - return policyRules - } - // collect all the namespaces and sort them var namespaces []string for ns := range rulesByNSResource { @@ -393,3 +292,104 @@ func (g Generator) Generate(ctx *genall.GenerationContext) error { return ctx.WriteYAML(fileName, headerText, objs, genall.WithTransform(genall.TransformRemoveCreationTimestamp)) } + +// NormalizeRules merge Rule with the same ruleKey and sort the Rules +func NormalizeRules(rules []*Rule) []rbacv1.PolicyRule { + ruleMap := make(map[ruleKey]*Rule) + // all the Rules having the same ruleKey will be merged into the first Rule + for _, rule := range rules { + // fix the group name first, since letting people type "core" is nice + for i, name := range rule.Groups { + if name == "core" { + rule.Groups[i] = "" + } + } + + key := rule.key() + if _, ok := ruleMap[key]; !ok { + ruleMap[key] = rule + continue + } + ruleMap[key].addVerbs(rule.Verbs) + } + + // deduplicate resources + // 1. create map based on key without resources + ruleMapWithoutResources := make(map[string][]*Rule) + for _, rule := range ruleMap { + // get key without Resources + key := rule.keyWithGroupResourceNamesURLsVerbs() + ruleMapWithoutResources[key] = append(ruleMapWithoutResources[key], rule) + } + // 2. merge to ruleMap + ruleMap = make(map[ruleKey]*Rule) + for _, rules := range ruleMapWithoutResources { + rule := rules[0] + for _, mergeRule := range rules[1:] { + rule.Resources = append(rule.Resources, mergeRule.Resources...) + } + + key := rule.key() + ruleMap[key] = rule + } + + // deduplicate groups + // 1. create map based on key without group + ruleMapWithoutGroup := make(map[string][]*Rule) + for _, rule := range ruleMap { + // get key without Group + key := rule.keyWithResourcesResourceNamesURLsVerbs() + ruleMapWithoutGroup[key] = append(ruleMapWithoutGroup[key], rule) + } + // 2. merge to ruleMap + ruleMap = make(map[ruleKey]*Rule) + for _, rules := range ruleMapWithoutGroup { + rule := rules[0] + for _, mergeRule := range rules[1:] { + rule.Groups = append(rule.Groups, mergeRule.Groups...) + } + key := rule.key() + ruleMap[key] = rule + } + + // deduplicate URLs + // 1. create map based on key without URLs + ruleMapWithoutURLs := make(map[string][]*Rule) + for _, rule := range ruleMap { + // get key without Group + key := rule.keyWitGroupResourcesResourceNamesVerbs() + ruleMapWithoutURLs[key] = append(ruleMapWithoutURLs[key], rule) + } + // 2. merge to ruleMap + ruleMap = make(map[ruleKey]*Rule) + for _, rules := range ruleMapWithoutURLs { + rule := rules[0] + for _, mergeRule := range rules[1:] { + rule.URLs = append(rule.URLs, mergeRule.URLs...) + } + key := rule.key() + ruleMap[key] = rule + } + + // sort the Rules in rules according to their ruleKeys + keys := make([]ruleKey, 0, len(ruleMap)) + for key := range ruleMap { + keys = append(keys, key) + } + sort.Sort(ruleKeys(keys)) + + // Normalize rule verbs to "*" if any verb in the rule is an asterisk + for _, rule := range ruleMap { + for _, verb := range rule.Verbs { + if verb == "*" { + rule.Verbs = []string{"*"} + break + } + } + } + var policyRules []rbacv1.PolicyRule + for _, key := range keys { + policyRules = append(policyRules, ruleMap[key].ToRule()) + } + return policyRules +}