Merge pull request #1916 from mrueg/config-change-metrics

Add metrics for config file changes

Author: k8s-ci-robot

README.md

@@ -165,6 +165,14 @@ please check the blog post [here](https://www.robustperception.io/exposing-the-s
 Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate
 run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules).
 
+kube-state-metrics also exposes metrics about it config file: 
+
+```
+kube_state_metrics_config_hash{type="config", filename="config.yml"} 4.0061079457904e+13
+kube_state_metrics_config_last_reload_success_timestamp_seconds{type="config", filename="config.yml"} 1.6697483049487052e+09
+kube_state_metrics_config_last_reload_successful{type="config", filename="config.yml"} 1
+```
+
 ### Scaling kube-state-metrics
 
 #### Resource recommendation

internal/wrapper.go

@@ -54,7 +54,7 @@ func RunKubeStateMetricsWrapper(opts *options.Options) {
 		}
 	}
 	ctx, cancel := context.WithCancel(context.Background())
-	if file := options.GetOptsConfigFile(*opts); file != "" {
+	if file := options.GetConfigFile(*opts); file != "" {
 		viper.SetConfigType("yaml")
 		viper.SetConfigFile(file)
 		if err := viper.ReadInConfig(); err != nil {

pkg/app/server.go

@@ -18,6 +18,8 @@ package app
 
 import (
 	"context"
+	"crypto/md5" //nolint:gosec
+	"encoding/binary"
 	"fmt"
 	"net"
 	"net/http"
@@ -99,24 +101,46 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options, factories .
 			ConstLabels: prometheus.Labels{"handler": "metrics"},
 		}, []string{"method"},
 	)
+	configHash := promauto.With(ksmMetricsRegistry).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "kube_state_metrics_config_hash",
+			Help: "Hash of the currently loaded configuration.",
+		}, []string{"type", "filename"})
+	configSuccess := promauto.With(ksmMetricsRegistry).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "kube_state_metrics_last_config_reload_successful",
+			Help: "Whether the last configuration reload attempt was successful.",
+		}, []string{"type", "filename"})
+	configSuccessTime := promauto.With(ksmMetricsRegistry).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "kube_state_metrics_last_config_reload_success_timestamp_seconds",
+			Help: "Timestamp of the last successful configuration reload.",
+		}, []string{"type", "filename"})
+
 	storeBuilder.WithMetrics(ksmMetricsRegistry)
 
-	got := options.GetOptsConfigFile(*opts)
+	got := options.GetConfigFile(*opts)
 	if got != "" {
-		optsConfigFile, err := os.ReadFile(filepath.Clean(got))
+		configFile, err := os.ReadFile(filepath.Clean(got))
 		if err != nil {
 			return fmt.Errorf("failed to read opts config file: %v", err)
 		}
 		// NOTE: Config value will override default values of intersecting options.
-		err = yaml.Unmarshal(optsConfigFile, opts)
+		err = yaml.Unmarshal(configFile, opts)
 		if err != nil {
 			// DO NOT end the process.
 			// We want to allow the user to still be able to fix the misconfigured config (redeploy or edit the configmaps) and reload KSM automatically once that's done.
 			klog.Warningf("failed to unmarshal opts config file: %v", err)
 			// Wait for the next reload.
 			klog.Infof("misconfigured config detected, KSM will automatically reload on next write to the config")
 			klog.Infof("waiting for config to be fixed")
+			configSuccess.WithLabelValues("config", filepath.Clean(got)).Set(0)
 			<-ctx.Done()
+		} else {
+			configSuccess.WithLabelValues("config", filepath.Clean(got)).Set(1)
+			configSuccessTime.WithLabelValues("config", filepath.Clean(got)).SetToCurrentTime()
+			hash := md5HashAsMetricValue(configFile)
+			configHash.WithLabelValues("config", filepath.Clean(got)).Set(hash)
 		}
 	}
 	var resources []string
@@ -371,3 +395,14 @@ func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prome
 	})
 	return mux
 }
+
+// md5HashAsMetricValue creates an md5 hash and returns the most significant bytes that fit into a float64
+// Taken from https://github.com/prometheus/alertmanager/blob/6ef6e6868dbeb7984d2d577dd4bf75c65bf1904f/config/coordinator.go#L149
+func md5HashAsMetricValue(data []byte) float64 {
+	sum := md5.Sum(data) //nolint:gosec
+	// We only want 48 bits as a float64 only has a 53 bit mantissa.
+	smallSum := sum[0:6]
+	bytes := make([]byte, 8)
+	copy(bytes, smallSum)
+	return float64(binary.LittleEndian.Uint64(bytes))
+}

pkg/options/options.go

@@ -62,8 +62,8 @@ type Options struct {
 	cmd *cobra.Command
 }
 
-// GetOptsConfigFile is the getter for --options-config-file value.
-func GetOptsConfigFile(opt Options) string {
+// GetConfigFile is the getter for --config value.
+func GetConfigFile(opt Options) string {
 	return opt.Config
 }