Merge pull request #2388 from CatherineF-dev/patch-11

fix: use --track-unscheduled-pods to select unscheduled pods in Daemonset sharding

Author: k8s-ci-robot

README.md

@@ -282,7 +282,7 @@ spec:
               fieldPath: spec.nodeName
 ```
 
-To track metrics for unassigned pods, you need to add an additional deployment and set `--node=""`, as shown in the following example:
+To track metrics for unassigned pods, you need to add an additional deployment and set `--track-unscheduled-pods`, as shown in the following example:
 
 ```
 apiVersion: apps/v1
@@ -295,7 +295,7 @@ spec:
         name: kube-state-metrics
         args:
         - --resources=pods
-        - --node=""
+        - --track-unscheduled-pods
 ```
 
 Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).

README.md.tpl

@@ -283,7 +283,7 @@ spec:
               fieldPath: spec.nodeName
 ```
 
-To track metrics for unassigned pods, you need to add an additional deployment and set `--node=""`, as shown in the following example:
+To track metrics for unassigned pods, you need to add an additional deployment and set `--track-unscheduled-pods`, as shown in the following example:
 
 ```
 apiVersion: apps/v1
@@ -296,7 +296,7 @@ spec:
         name: kube-state-metrics
         args:
         - --resources=pods
-        - --node=""
+        - --track-unscheduled-pods
 ```
 
 Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).

docs/developer/cli-arguments.md

@@ -81,6 +81,7 @@ Flags:
       --telemetry-port int                         Port to expose kube-state-metrics self metrics on. (default 8081)
       --tls-config string                          Path to the TLS configuration file
       --total-shards int                           The total number of shards. Sharding is disabled when total shards is set to 1. (default 1)
+      --track-unscheduled-pods                     This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of unscheduled pods is fetched from the Kubernetes API Server. This is experimental.
       --use-apiserver-cache                        Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.
   -v, --v Level                                    number for the log level verbosity
       --vmodule moduleSpec                         comma-separated list of pattern=N settings for file-filtered logging

examples/daemonsetsharding/deployment-no-node-pods-service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics-no-node-pods
+    app.kubernetes.io/version: 2.12.0
+  name: kube-state-metrics-no-node-pods
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+  - name: http-metrics
+    port: 8080
+    targetPort: http-metrics
+  - name: telemetry
+    port: 8081
+    targetPort: telemetry
+  selector:
+    app.kubernetes.io/name: kube-state-metrics-no-node-pods

examples/daemonsetsharding/deployment-no-node-pods.yaml

@@ -3,35 +3,35 @@ kind: Deployment
 metadata:
   labels:
     app.kubernetes.io/component: exporter
-    app.kubernetes.io/name: kube-state-metrics-pods
+    app.kubernetes.io/name: kube-state-metrics-no-node-pods
     app.kubernetes.io/version: 2.13.0
-  name: kube-state-metrics-pods
+  name: kube-state-metrics-no-node-pods
   namespace: kube-system
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: kube-state-metrics
+      app.kubernetes.io/name: kube-state-metrics-no-node-pods
   template:
     metadata:
       labels:
         app.kubernetes.io/component: exporter
-        app.kubernetes.io/name: kube-state-metrics
+        app.kubernetes.io/name: kube-state-metrics-no-node-pods
         app.kubernetes.io/version: 2.13.0
     spec:
       automountServiceAccountToken: true
       containers:
       - args:
         - --resources=pods
-        - --node=""
+        - --track-unscheduled-pods
         image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
         livenessProbe:
           httpGet:
             path: /livez
             port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
-        name: kube-state-metrics
+        name: kube-state-metrics-no-node-pods
         ports:
         - containerPort: 8080
           name: http-metrics

jsonnet/kube-state-metrics/kube-state-metrics.libsonnet

@@ -377,18 +377,27 @@
       local c = ksm.deployment.spec.template.spec.containers[0] {
         args: [
           '--resources=pods',
-          '--node=""',
+          '--track-unscheduled-pods',
         ],
+        name: shardksmname,
       };
-      local shardksmname = ksm.name + "-pods";
+      local shardksmname = ksm.name + "-unscheduled-pods-fetching";
       std.mergePatch(ksm.deployment,
         {
           metadata: {
             name: shardksmname,
             labels: {'app.kubernetes.io/name': shardksmname}
           },
           spec: {
+            selector{
+              matchLabels: {app.kubernetes.io/name': shardksmname}
+            }
             template: {
+              metadata: {
+                labels: {
+                  app.kubernetes.io/name': shardksmname
+                }
+              }
               spec: {
                 containers: [c],
               },
@@ -397,6 +406,27 @@
         },
       ),
 
+    deploymentNoNodePodsService:
+      local c = ksm.deployment.spec.template.spec.containers[0] {
+        args: [
+          '--resources=pods',
+          '--track-unscheduled-pods',
+        ],
+      };
+      local shardksmname = ksm.name + "-no-node-pods";
+      std.mergePatch(ksm.service,
+        {
+          metadata: {
+            name: shardksmname,
+            labels: {'app.kubernetes.io/name': shardksmname}
+          },
+          spec: {
+            selector: {
+              'app.kubernetes.io/name': shardksmname
+            }
+          }
+        }
+      ),
     daemonset:
       // extending the default container from above
       local c0 = ksm.deployment.spec.template.spec.containers[0] {

pkg/app/server.go

@@ -224,7 +224,13 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
 
 	namespaces := opts.Namespaces.GetNamespaces()
 	nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
-	nodeFieldSelector := opts.Node.GetNodeFieldSelector()
+	var nodeFieldSelector string
+	if opts.TrackUnscheduledPods {
+		nodeFieldSelector = "spec.nodeName="
+		klog.InfoS("Using spec.nodeName= to select unscheduable pods without node")
+	} else {
+		nodeFieldSelector = opts.Node.GetNodeFieldSelector()
+	}
 	merged, err := storeBuilder.MergeFieldSelectors([]string{nsFieldSelector, nodeFieldSelector})
 	if err != nil {
 		return err

pkg/options/options.go

@@ -59,6 +59,7 @@ type Options struct {
 	Namespaces               NamespaceList   `yaml:"namespaces"`
 	NamespacesDenylist       NamespaceList   `yaml:"namespaces_denylist"`
 	Node                     NodeType        `yaml:"node"`
+	TrackUnscheduledPods     bool            `yaml:"track_unscheduled_pods"`
 	Pod                      string          `yaml:"pod"`
 	Port                     int             `yaml:"port"`
 	Resources                ResourceSet     `yaml:"resources"`
@@ -90,7 +91,6 @@ func NewOptions() *Options {
 		MetricAllowlist:      MetricSet{},
 		MetricDenylist:       MetricSet{},
 		MetricOptInList:      MetricSet{},
-		Node:                 NodeType{},
 		AnnotationsAllowList: LabelsAllowList{},
 		LabelsAllowList:      LabelsAllowList{},
 	}
@@ -138,6 +138,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 
 	o.cmd.Flags().BoolVar(&o.CustomResourcesOnly, "custom-resource-state-only", false, "Only provide Custom Resource State metrics (experimental)")
 	o.cmd.Flags().BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.")
+	o.cmd.Flags().BoolVar(&o.TrackUnscheduledPods, "track-unscheduled-pods", false, "This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of unscheduled pods is fetched from the Kubernetes API Server. This is experimental.")
 	o.cmd.Flags().BoolVarP(&o.Help, "help", "h", false, "Print Help text")
 	o.cmd.Flags().BoolVarP(&o.UseAPIServerCache, "use-apiserver-cache", "", false, "Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.")
 	o.cmd.Flags().Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)")
@@ -156,7 +157,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file")
 	o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
 	o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file")
-	o.cmd.Flags().Var(&o.Node, "node", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
+	o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
 	o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
 	o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.")
 	o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
@@ -186,7 +187,7 @@ func (o *Options) Usage() {
 // Validate validates arguments
 func (o *Options) Validate() error {
 	shardableResource := "pods"
-	if o.Node.String() == "" {
+	if o.Node == "" {
 		return nil
 	}
 	for _, x := range o.Resources.AsSlice() {

pkg/options/types.go

@@ -18,7 +18,6 @@ package options
 
 import (
 	"errors"
-	"regexp"
 	"sort"
 	"strings"
 
@@ -106,56 +105,25 @@ func (r *ResourceSet) Type() string {
 }
 
 // NodeType represents a nodeName to query from.
-type NodeType map[string]struct{}
+type NodeType string
 
-// Set converts a comma-separated string of nodename into a slice and appends it to the NodeList
+// Set sets the node name to NodeType.
 func (n *NodeType) Set(value string) error {
-	s := *n
-	cols := strings.Split(value, ",")
-	for _, col := range cols {
-		col = strings.TrimSpace(col)
-		if len(col) != 0 {
-			s[col] = struct{}{}
-		}
-	}
+	*n = NodeType(value)
 	return nil
 }
 
-// AsSlice returns the LabelsAllowList in the form of plain string slice.
-func (n NodeType) AsSlice() []string {
-	cols := make([]string, 0, len(n))
-	for col := range n {
-		cols = append(cols, col)
-	}
-	return cols
-}
-
+// String gets node name.
 func (n NodeType) String() string {
-	return strings.Join(n.AsSlice(), ",")
-}
-
-// Type returns a descriptive string about the NodeList type.
-func (n *NodeType) Type() string {
-	return "string"
+	return string(n)
 }
 
 // GetNodeFieldSelector returns a nodename field selector.
 func (n *NodeType) GetNodeFieldSelector() string {
-	if nil == n || len(*n) == 0 {
-		klog.InfoS("Using node type is nil")
-		return EmptyFieldSelector()
+	if string(*n) != "" {
+		return fields.OneTermEqualSelector("spec.nodeName", string(*n)).String()
 	}
-	pattern := "[^a-zA-Z0-9_,-]+"
-	re := regexp.MustCompile(pattern)
-	result := re.ReplaceAllString(n.String(), "")
-	klog.InfoS("Using node type", "node", result)
-	return fields.OneTermEqualSelector("spec.nodeName", result).String()
-
-}
-
-// NodeValue represents a nodeName to query from.
-type NodeValue interface {
-	GetNodeFieldSelector() string
+	return EmptyFieldSelector()
 }
 
 // EmptyFieldSelector returns an empty field selector.