diff --git a/internal/store/builder.go b/internal/store/builder.go index b66e455dc2..1f794b046e 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -75,6 +75,7 @@ type Builder struct { buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc allowAnnotationsList map[string][]string allowLabelsList map[string][]string + injectPerSeriesMetadata bool utilOptions *options.Options // namespaceFilter is inside fieldSelectorFilter fieldSelectorFilter string @@ -245,6 +246,13 @@ func (b *Builder) allowList(list map[string][]string) (map[string][]string, erro return m, nil } +// WithAllowAnnotations configures which annotations can be returned for metrics +func (b *Builder) WithInjectPerSeriesMetadata(inject bool) error { + var err error + b.injectPerSeriesMetadata = inject + return err +} + // WithAllowAnnotations configures which annotations can be returned for metrics func (b *Builder) WithAllowAnnotations(annotations map[string][]string) error { var err error @@ -463,7 +471,7 @@ func (b *Builder) buildStorageClassStores() []cache.Store { } func (b *Builder) buildPodStores() []cache.Store { - return b.buildStoresFunc(podMetricFamilies(b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache) + return b.buildStoresFunc(podMetricFamilies(b.injectPerSeriesMetadata, b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache) } func (b *Builder) buildCsrStores() []cache.Store { diff --git a/internal/store/pod.go b/internal/store/pod.go index d9ad4afa6a..e666c799f6 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -40,7 +40,12 @@ var ( podStatusReasons = []string{"Evicted", "NodeAffinity", "NodeLost", "Shutdown", "UnexpectedAdmissionError"} ) -func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { +func podMetricFamilies(injectPerSeriesMetadata bool, allowAnnotationsList []string, allowLabelsList []string) []generator.FamilyGenerator { + mc := &MetricConfig{ + InjectPerSeriesMetadata: injectPerSeriesMetadata, + AllowAnnotations: allowAnnotationsList, + AllowLabels: allowLabelsList, + } return []generator.FamilyGenerator{ createPodCompletionTimeFamilyGenerator(), createPodContainerInfoFamilyGenerator(), @@ -82,7 +87,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodSpecVolumesPersistentVolumeClaimsInfoFamilyGenerator(), createPodSpecVolumesPersistentVolumeClaimsReadonlyFamilyGenerator(), createPodStartTimeFamilyGenerator(), - createPodStatusPhaseFamilyGenerator(), + createPodStatusPhaseFamilyGenerator(mc), createPodStatusQosClassFamilyGenerator(), createPodStatusReadyFamilyGenerator(), createPodStatusReadyTimeFamilyGenerator(), @@ -1333,7 +1338,7 @@ func createPodStartTimeFamilyGenerator() generator.FamilyGenerator { ) } -func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { +func createPodStatusPhaseFamilyGenerator(mc *MetricConfig) generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_status_phase", "The pods current phase.", @@ -1361,13 +1366,12 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { ms := make([]*metric.Metric, len(phases)) - for i, p := range phases { - ms[i] = &metric.Metric{ - + for i, ph := range phases { + ms[i] = injectLabelsAndAnnos(&metric.Metric{ LabelKeys: []string{"phase"}, - LabelValues: []string{p.n}, - Value: boolFloat64(p.v), - } + LabelValues: []string{ph.n}, + Value: boolFloat64(ph.v), + }, mc, &p.ObjectMeta) } return &metric.Family{ diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 3ba2875d01..ccc4f2ca41 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -2181,8 +2181,8 @@ func TestPodStore(t *testing.T) { } for i, c := range cases { - c.Func = generator.ComposeMetricGenFuncs(podMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) - c.Headers = generator.ExtractMetricFamilyHeaders(podMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Func = generator.ComposeMetricGenFuncs(podMetricFamilies(false, c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(podMetricFamilies(false, c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } @@ -2192,7 +2192,7 @@ func TestPodStore(t *testing.T) { func BenchmarkPodStore(b *testing.B) { b.ReportAllocs() - f := generator.ComposeMetricGenFuncs(podMetricFamilies(nil, nil)) + f := generator.ComposeMetricGenFuncs(podMetricFamilies(false, nil, nil)) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/store/utils.go b/internal/store/utils.go index 5cebe03ca4..2c64cae974 100644 --- a/internal/store/utils.go +++ b/internal/store/utils.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/kube-state-metrics/v2/pkg/metric" @@ -38,6 +39,12 @@ var ( conditionStatuses = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown} ) +type MetricConfig struct { + InjectPerSeriesMetadata bool + AllowAnnotations []string + AllowLabels []string +} + func resourceVersionMetric(rv string) []*metric.Metric { v, err := strconv.ParseFloat(rv, 64) if err != nil { @@ -175,6 +182,17 @@ func isPrefixedNativeResource(name v1.ResourceName) bool { return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix) } +// convenience wrapper to inject allow-listed labels and annotations to a metric if per-series injection is enabled. +func injectLabelsAndAnnos(m *metric.Metric, metricConfig *MetricConfig, obj *metav1.ObjectMeta) *metric.Metric { + if !metricConfig.InjectPerSeriesMetadata { + return m + } + labelKeys, labelValues := createPrometheusLabelKeysValues("label", obj.Labels, metricConfig.AllowLabels) + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", obj.Annotations, metricConfig.AllowAnnotations) + m.LabelKeys, m.LabelValues = mergeKeyValues(m.LabelKeys, m.LabelValues, annotationKeys, annotationValues, labelKeys, labelValues) + return m +} + // createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels // and associated allowed list in kubernetes label format. // It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels. diff --git a/pkg/app/server.go b/pkg/app/server.go index 8ab3ad54f8..d7e40b52d3 100644 --- a/pkg/app/server.go +++ b/pkg/app/server.go @@ -264,6 +264,9 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { if err := storeBuilder.WithAllowLabels(opts.LabelsAllowList); err != nil { return fmt.Errorf("failed to set up labels allowlist: %v", err) } + if err := storeBuilder.WithInjectPerSeriesMetadata(opts.InjectPerSeriesMetadata); err != nil { + return fmt.Errorf("failed to configure per series metadata injection: %v", err) + } ksmMetricsRegistry.MustRegister( collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), diff --git a/pkg/options/options.go b/pkg/options/options.go index 95af22b55f..03869aa504 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -47,6 +47,8 @@ type Options struct { MetricOptInList MetricSet `yaml:"metric_opt_in_list"` Resources ResourceSet `yaml:"resources"` + InjectPerSeriesMetadata bool `yaml:"inject_per_series_metadata"` + cmd *cobra.Command Apiserver string `yaml:"apiserver"` CustomResourceConfig string `yaml:"custom_resource_config"` @@ -164,6 +166,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.") o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.") o.cmd.Flags().Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.") + o.cmd.Flags().BoolVar(&o.InjectPerSeriesMetadata, "inject-per-series-metadata", false, "Propagate labels and annotations from object metadata to all respective metrics rather than simply populating kube__labels and kube__annotations. Honors metric-annotations-allowlist and metric-labels-allowlist.") o.cmd.Flags().Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists") o.cmd.Flags().Var(&o.Namespaces, "namespaces", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces)) o.cmd.Flags().Var(&o.NamespacesDenylist, "namespaces-denylist", "Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.")