Add end-to-end testing to verify hot-reloading

Add end-to-end testing to verify hot-reloading for event-based changes
stemming from the config file. Also,
* sent in a doc fix that was missed
earlier:
https://github.com/kubernetes/kube-state-metrics/pull/1890/files#diff-380eca5a922c0ddbf67f04daefc6823e7ef0e197434d3a826d39c7063cdfa6d6R15,
* updated fsnotify and viper dependencies (v1.6.0 and v1.14.0
  respectively).

Signed-off-by: Pranshu Srivastava <[email protected]>

Author: rexagod

docs/node-metrics.md

@@ -12,4 +12,4 @@
 | kube_node_status_allocatable | Gauge | The allocatable for different resources of a node that are available for scheduling | `cpu`=&lt;core&gt; <br> `ephemeral_storage`=&lt;byte&gt; <br> `pods`=&lt;integer&gt; <br> `attachable_volumes_*`=&lt;byte&gt; <br> `hugepages_*`=&lt;byte&gt; <br> `memory`=&lt;byte&gt; |`node`=&lt;node-address&gt; <br> `resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt;| STABLE |
 | kube_node_status_condition | Gauge | The condition of a cluster node | |`node`=&lt;node-address&gt; <br> `condition`=&lt;node-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt; | STABLE |
 | kube_node_created | Gauge | Unix creation timestamp | seconds |`node`=&lt;node-address&gt;| STABLE |
-| kube_node_deletion_timestamp | Gauge | Unix creation timestamp | seconds |`node`=&lt;node-address&gt;| STABLE |
+| kube_node_deletion_timestamp | Gauge | Unix creation timestamp | seconds |`node`=&lt;node-address&gt;| EXPERIMENTAL |

go.mod

@@ -4,7 +4,7 @@ require (
 	github.com/brancz/gojsontoyaml v0.1.0
 	github.com/campoy/embedmd v1.0.0
 	github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0
-	github.com/fsnotify/fsnotify v1.5.4
+	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gobuffalo/flect v0.3.0
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-jsonnet v0.19.1
@@ -16,7 +16,7 @@ require (
 	github.com/prometheus/exporter-toolkit v0.8.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/cobra v1.6.1
-	github.com/spf13/viper v1.13.0
+	github.com/spf13/viper v1.14.0
 	github.com/stretchr/testify v1.8.1
 	golang.org/x/perf v0.0.0-20220920022801-e8d778a60d07
 	gopkg.in/yaml.v3 v3.0.1
@@ -31,7 +31,8 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
@@ -84,19 +85,19 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect
-	golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
-	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
+	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.1.0 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect

go.sum

@@ -0,0 +1,96 @@
+/*
+Copyright 2022 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/spf13/viper"
+	"gopkg.in/yaml.v3"
+	"k8s.io/klog/v2"
+
+	"k8s.io/kube-state-metrics/v2/pkg/app"
+	"k8s.io/kube-state-metrics/v2/pkg/customresource"
+	"k8s.io/kube-state-metrics/v2/pkg/customresourcestate"
+	"k8s.io/kube-state-metrics/v2/pkg/options"
+)
+
+// RunKubeStateMetricsWrapper is a wrapper around KSM, delegated to the root command.
+func RunKubeStateMetricsWrapper(opts *options.Options) {
+	var factories []customresource.RegistryFactory
+	if config, set := resolveCustomResourceConfig(opts); set {
+		crf, err := customresourcestate.FromConfig(config)
+		if err != nil {
+			klog.ErrorS(err, "Parsing from Custom Resource State Metrics file failed")
+			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+		}
+		factories = append(factories, crf...)
+	}
+
+	KSMRunOrDie := func(ctx context.Context) {
+		if err := app.RunKubeStateMetricsWrapper(ctx, opts, factories...); err != nil {
+			klog.ErrorS(err, "Failed to run kube-state-metrics")
+			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+		}
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	if file := options.GetOptsConfigFile(*opts); file != "" {
+		viper.SetConfigType("yaml")
+		viper.SetConfigFile(file)
+		if err := viper.ReadInConfig(); err != nil {
+			if errors.Is(err, viper.ConfigFileNotFoundError{}) {
+				klog.ErrorS(err, "Options configuration file not found", "file", file)
+			} else {
+				klog.ErrorS(err, "Error reading options configuration file", "file", file)
+			}
+			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+		}
+		viper.OnConfigChange(func(e fsnotify.Event) {
+			klog.Infof("Changes detected: %s\n", e.Name)
+			cancel()
+			// Wait for the ports to be released.
+			<-time.After(3 * time.Second)
+			ctx, cancel = context.WithCancel(context.Background())
+			go KSMRunOrDie(ctx)
+		})
+		viper.WatchConfig()
+	}
+	klog.Infoln("Starting kube-state-metrics")
+	KSMRunOrDie(ctx)
+	select {}
+}
+
+func resolveCustomResourceConfig(opts *options.Options) (customresourcestate.ConfigDecoder, bool) {
+	if s := opts.CustomResourceConfig; s != "" {
+		return yaml.NewDecoder(strings.NewReader(s)), true
+	}
+	if file := opts.CustomResourceConfigFile; file != "" {
+		f, err := os.Open(filepath.Clean(file))
+		if err != nil {
+			klog.ErrorS(err, "Custom Resource State Metrics file could not be opened")
+			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+		}
+		return yaml.NewDecoder(f), true
+	}
+	return nil, false
+}

internal/wrapper.go

@@ -17,30 +17,18 @@ limitations under the License.
 package main
 
 import (
-	"context"
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"gopkg.in/yaml.v3"
 	"k8s.io/klog/v2"
 
-	"k8s.io/kube-state-metrics/v2/pkg/app"
-	"k8s.io/kube-state-metrics/v2/pkg/customresource"
-	"k8s.io/kube-state-metrics/v2/pkg/customresourcestate"
+	"k8s.io/kube-state-metrics/v2/internal"
 	"k8s.io/kube-state-metrics/v2/pkg/options"
 )
 
 func main() {
 	opts := options.NewOptions()
 	cmd := options.InitCommand
 	cmd.Run = func(cmd *cobra.Command, args []string) {
-		RunKubeStateMetricsWrapper(opts)
+		internal.RunKubeStateMetricsWrapper(opts)
 	}
 	opts.AddFlags(cmd)
 
@@ -53,63 +41,3 @@ func main() {
 		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
 	}
 }
-
-// RunKubeStateMetricsWrapper is a wrapper around KSM, delegated to the root command.
-func RunKubeStateMetricsWrapper(opts *options.Options) {
-	var factories []customresource.RegistryFactory
-	if config, set := resolveCustomResourceConfig(opts); set {
-		crf, err := customresourcestate.FromConfig(config)
-		if err != nil {
-			klog.ErrorS(err, "Parsing from Custom Resource State Metrics file failed")
-			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
-		}
-		factories = append(factories, crf...)
-	}
-
-	KSMRunOrDie := func(ctx context.Context) {
-		if err := app.RunKubeStateMetricsWrapper(ctx, opts, factories...); err != nil {
-			klog.ErrorS(err, "Failed to run kube-state-metrics")
-			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
-		}
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	if file := options.GetOptsConfigFile(*opts); file != "" {
-		viper.SetConfigType("yaml")
-		viper.SetConfigFile(file)
-		if err := viper.ReadInConfig(); err != nil {
-			if errors.Is(err, viper.ConfigFileNotFoundError{}) {
-				klog.ErrorS(err, "Options configuration file not found", "file", file)
-			} else {
-				klog.ErrorS(err, "Error reading options configuration file", "file", file)
-			}
-			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
-		}
-		viper.OnConfigChange(func(e fsnotify.Event) {
-			klog.Infof("Changes detected: %s\n", e.Name)
-			cancel()
-			// Wait for the ports to be released.
-			<-time.After(3 * time.Second)
-			ctx, cancel = context.WithCancel(context.Background())
-			go KSMRunOrDie(ctx)
-		})
-		viper.WatchConfig()
-	}
-	klog.Infoln("Starting kube-state-metrics")
-	KSMRunOrDie(ctx)
-	select {}
-}
-
-func resolveCustomResourceConfig(opts *options.Options) (customresourcestate.ConfigDecoder, bool) {
-	if s := opts.CustomResourceConfig; s != "" {
-		return yaml.NewDecoder(strings.NewReader(s)), true
-	}
-	if file := opts.CustomResourceConfigFile; file != "" {
-		f, err := os.Open(filepath.Clean(file))
-		if err != nil {
-			klog.ErrorS(err, "Custom Resource State Metrics file could not be opened")
-			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
-		}
-		return yaml.NewDecoder(f), true
-	}
-	return nil, false
-}

main.go

@@ -57,14 +57,14 @@ type Options struct {
 	UseAPIServerCache        bool            `yaml:"use_api_server_cache"`
 	Version                  bool            `yaml:"version"`
 
-	optsConfigFile string
+	Config string
 
 	cmd *cobra.Command
 }
 
 // GetOptsConfigFile is the getter for --options-config-file value.
 func GetOptsConfigFile(opt Options) string {
-	return opt.optsConfigFile
+	return opt.Config
 }
 
 // NewOptions returns a new instance of `Options`.
@@ -137,7 +137,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	o.cmd.Flags().StringVar(&o.Pod, "pod", "", "Name of the pod that contains the kube-state-metrics container. "+autoshardingNotice)
 	o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file")
 	o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
-	o.cmd.Flags().StringVar(&o.optsConfigFile, "config", "", "Path to the kube-state-metrics options config file")
+	o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file")
 	o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
 	o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
 	o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.")

pkg/options/options.go

@@ -48,14 +48,18 @@ func TestOptionsParse(t *testing.T) {
 	opts.AddFlags(InitCommand)
 
 	for _, test := range tests {
+		t.Run(test.Desc, func(t *testing.T) {
+			os.Args = test.Args
 
-		os.Args = test.Args
+			err := opts.Parse()
 
-		err := opts.Parse()
-		if err != nil {
-			if !test.ExpectsError {
+			if !test.ExpectsError && err != nil {
 				t.Errorf("Error for test with description: %s: %v", test.Desc, err.Error())
 			}
-		}
+
+			if test.ExpectsError && err == nil {
+				t.Errorf("Expected error for test with description: %s", test.Desc)
+			}
+		})
 	}
 }

pkg/options/options_test.go

@@ -155,7 +155,8 @@ echo "kube-state-metrics is up and running"
 echo "start e2e test for kube-state-metrics"
 KSM_HTTP_METRICS_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy'
 KSM_TELEMETRY_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:telemetry/proxy'
-go test -v ./tests/e2e/ --ksm-http-metrics-url=${KSM_HTTP_METRICS_URL} --ksm-telemetry-url=${KSM_TELEMETRY_URL}
+go test -v ./tests/e2e/main_test.go --ksm-http-metrics-url=${KSM_HTTP_METRICS_URL} --ksm-telemetry-url=${KSM_TELEMETRY_URL}
+go test -v ./tests/e2e/hot-reload_test.go
 
 mkdir -p ${KUBE_STATE_METRICS_LOG_DIR}