Merge pull request #2705 from rashmichandrashekar/rashmi/ksm-config-override

fix: Config file overrides apply to some fields but not other

Author: k8s-ci-robot

docs/developer/cli-arguments.md

@@ -44,7 +44,7 @@ Flags:
       --auth-filter                                If true, requires authentication and authorization through Kubernetes API to access metrics endpoints
       --auto-gomemlimit                            Automatically set GOMEMLIMIT to match container or system memory limit. (experimental)
       --auto-gomemlimit-ratio float                The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory. (experimental) (default 0.9)
-      --config string                              Path to the kube-state-metrics options config file
+      --config string                              Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.
       --custom-resource-state-config string        Inline Custom Resource State Metrics config YAML (experimental)
       --custom-resource-state-config-file string   Path to a Custom Resource State Metrics config file (experimental)
       --custom-resource-state-only                 Only provide Custom Resource State metrics (experimental)

pkg/app/server.go

@@ -149,6 +149,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
 			hash := md5HashAsMetricValue(configFile)
 			configHash.WithLabelValues("config", filepath.Clean(got)).Set(hash)
 		}
+		opts = configureResourcesAndMetrics(opts, configFile)
 	}
 
 	if opts.AutoGoMemlimit {
@@ -264,9 +265,12 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
 	if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil {
 		return fmt.Errorf("failed to set up annotations allowlist: %v", err)
 	}
+	klog.InfoS("Using annotations allowlist", "annotationsAllowList", opts.AnnotationsAllowList)
+
 	if err := storeBuilder.WithAllowLabels(opts.LabelsAllowList); err != nil {
 		return fmt.Errorf("failed to set up labels allowlist: %v", err)
 	}
+	klog.InfoS("Using labels allowlist", "labelsAllowList", opts.LabelsAllowList)
 
 	ksmMetricsRegistry.MustRegister(
 		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
@@ -378,6 +382,59 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
 	return nil
 }
 
+func configureResourcesAndMetrics(opts *options.Options, configFile []byte) *options.Options {
+	// If the config file is set, we will overwrite the opts with the config file.
+	// This is only needed for maps because the default behaviour of yaml.Unmarshal is to append keys (and overwrite any conflicting ones).
+	config := options.NewOptions()
+	err := yaml.Unmarshal(configFile, &config)
+	if err == nil {
+		if len(config.Resources) > 0 {
+			opts.Resources = options.ResourceSet{}
+			for resource := range config.Resources {
+				opts.Resources[resource] = struct{}{}
+			}
+		}
+
+		if len(config.MetricAllowlist) > 0 {
+			opts.MetricAllowlist = options.MetricSet{}
+			for metric := range config.MetricAllowlist {
+				opts.MetricAllowlist[metric] = struct{}{}
+			}
+		}
+
+		if len(config.MetricDenylist) > 0 {
+			opts.MetricDenylist = options.MetricSet{}
+			for metric := range config.MetricDenylist {
+				opts.MetricDenylist[metric] = struct{}{}
+			}
+		}
+
+		if len(config.MetricOptInList) > 0 {
+			opts.MetricOptInList = options.MetricSet{}
+			for metric := range config.MetricOptInList {
+				opts.MetricOptInList[metric] = struct{}{}
+			}
+		}
+
+		if len(config.LabelsAllowList) > 0 {
+			opts.LabelsAllowList = options.LabelsAllowList{}
+			for label, value := range config.LabelsAllowList {
+				opts.LabelsAllowList[label] = value
+			}
+		}
+
+		if len(config.AnnotationsAllowList) > 0 {
+			opts.AnnotationsAllowList = options.LabelsAllowList{}
+			for annotation, value := range config.AnnotationsAllowList {
+				opts.AnnotationsAllowList[annotation] = value
+			}
+		}
+	} else {
+		klog.ErrorS(err, "failed to unmarshal configFile")
+	}
+	return opts
+}
+
 func buildTelemetryServer(registry prometheus.Gatherer, authFilter bool, kubeConfig *rest.Config) *http.ServeMux {
 	mux := http.NewServeMux()
 

pkg/app/server_test.go

@@ -981,3 +981,96 @@ func (f *fooFactory) ListWatch(customResourceClient interface{}, ns string, fiel
 		},
 	}
 }
+func TestConfigureResourcesAndMetrics(t *testing.T) {
+	// Prepare a config file in YAML format
+	configYAML := `
+"resources":
+  "pod": {}
+  "service": {}
+"metric_allowlist":
+  "kube_pod_info": {}
+"metric_denylist":
+  "kube_pod_labels": {}
+"metric_opt_in_list":
+  "kube_pod_status_phase": {}
+"labels_allow_list":
+  "labelX": 
+    - foo 
+    - bar
+"annotations_allow_list":
+  "annotationY": 
+     - baz
+`
+	opts := options.NewOptions()
+	// Set some initial values to be overwritten
+	opts.Resources = options.ResourceSet{"oldresource": {}}
+	opts.MetricAllowlist = options.MetricSet{"oldallow": {}}
+	opts.MetricDenylist = options.MetricSet{"olddeny": {}}
+	opts.MetricOptInList = options.MetricSet{"oldoptin": {}}
+	opts.LabelsAllowList = options.LabelsAllowList{"oldlabel": {"oldvalue"}}
+	opts.AnnotationsAllowList = options.LabelsAllowList{"oldannotation": {"oldvalue"}}
+
+	newOpts := configureResourcesAndMetrics(opts, []byte(configYAML))
+
+	// Check resources
+	expectedResources := []string{"pod", "service"}
+	for _, r := range expectedResources {
+		if _, ok := newOpts.Resources[r]; !ok {
+			t.Errorf("expected resource %q in opts.Resources", r)
+		}
+	}
+	if _, ok := newOpts.Resources["oldresource"]; ok {
+		t.Errorf("expected oldresource to be overwritten")
+	}
+
+	// Check metric allowlist
+	if _, ok := newOpts.MetricAllowlist["kube_pod_info"]; !ok {
+		t.Errorf("expected kube_pod_info in MetricAllowlist")
+	}
+	if _, ok := newOpts.MetricAllowlist["oldallow"]; ok {
+		t.Errorf("expected oldallow to be overwritten")
+	}
+
+	// Check metric denylist
+	if _, ok := newOpts.MetricDenylist["kube_pod_labels"]; !ok {
+		t.Errorf("expected kube_pod_labels in MetricDenylist")
+	}
+	if _, ok := newOpts.MetricDenylist["olddeny"]; ok {
+		t.Errorf("expected olddeny to be overwritten")
+	}
+
+	// Check metric opt-in list
+	if _, ok := newOpts.MetricOptInList["kube_pod_status_phase"]; !ok {
+		t.Errorf("expected kube_pod_status_phase in MetricOptInList")
+	}
+	if _, ok := newOpts.MetricOptInList["oldoptin"]; ok {
+		t.Errorf("expected oldoptin to be overwritten")
+	}
+
+	// Check labels allow list
+	if vals, ok := newOpts.LabelsAllowList["labelX"]; !ok || len(vals) != 2 || vals[0] != "foo" || vals[1] != "bar" {
+		t.Errorf("expected labelX with values [foo bar], got %v", vals)
+	}
+	if vals, ok := newOpts.LabelsAllowList["oldlabel"]; ok {
+		t.Errorf("expected oldlabel to be overwritten, got %v", vals)
+	}
+
+	// Check annotations allow list
+	if vals, ok := newOpts.AnnotationsAllowList["annotationY"]; !ok || len(vals) != 1 || vals[0] != "baz" {
+		t.Errorf("expected annotationY with value [baz], got %v", vals)
+	}
+	if vals, ok := newOpts.AnnotationsAllowList["oldannotation"]; ok {
+		t.Errorf("expected oldannotation to be overwritten, got %v", vals)
+	}
+
+}
+
+func TestConfigureResourcesAndMetrics_InvalidYAML(t *testing.T) {
+	opts := options.NewOptions()
+	invalidYAML := []byte("invalid: [unclosed")
+	// Should not panic or overwrite opts
+	result := configureResourcesAndMetrics(opts, invalidYAML)
+	if result != opts {
+		t.Errorf("expected opts to be returned unchanged on invalid YAML")
+	}
+}

pkg/options/options.go

@@ -162,7 +162,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	o.cmd.Flags().StringVar(&o.Pod, "pod", "", "Name of the pod that contains the kube-state-metrics container. "+autoshardingNotice)
 	o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file")
 	o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
-	o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file")
+	o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.")
 	o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
 	o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
 	o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.")