Skip to content

Commit b27845d

Browse files
CatherineF-devLaikaN57
CatherineF-dev
and
LaikaN57
committed
use --enable-no-node-scrape
Co-authored-by: Alex Kennedy <[email protected]>
1 parent 8f3c5c4 commit b27845d

File tree

11 files changed

+60
-133
lines changed

11 files changed

+60
-133
lines changed

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ spec:
282282
fieldPath: spec.nodeName
283283
```
284284

285-
To track metrics for unassigned pods, you need to add an additional deployment and set `--node=`, as shown in the following example:
285+
To track metrics for unassigned pods, you need to add an additional deployment and set `--enable-no-node-scrape`, as shown in the following example:
286286

287287
```
288288
apiVersion: apps/v1
@@ -295,7 +295,7 @@ spec:
295295
name: kube-state-metrics
296296
args:
297297
- --resources=pods
298-
- --node=""
298+
- --enable-no-node-scrape
299299
```
300300

301301
Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).

‎README.md.tpl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ spec:
283283
fieldPath: spec.nodeName
284284
```
285285

286-
To track metrics for unassigned pods, you need to add an additional deployment and set `--node=`, as shown in the following example:
286+
To track metrics for unassigned pods, you need to add an additional deployment and set `--enable-no-node-scrape`, as shown in the following example:
287287

288288
```
289289
apiVersion: apps/v1
@@ -296,7 +296,7 @@ spec:
296296
name: kube-state-metrics
297297
args:
298298
- --resources=pods
299-
- --node=""
299+
- --enable-no-node-scrape
300300
```
301301

302302
Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).

‎docs/developer/cli-arguments.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ Flags:
4848
--custom-resource-state-config-file string Path to a Custom Resource State Metrics config file (experimental)
4949
--custom-resource-state-only Only provide Custom Resource State metrics (experimental)
5050
--enable-gzip-encoding Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.
51+
--enable-no-node-scrape This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of no scheduled pods is scraped. This is experimental.
5152
-h, --help Print Help text
5253
--host string Host to expose metrics on. (default "::")
5354
--kubeconfig string Absolute path to the kubeconfig file

‎examples/daemonsetsharding/deployment-no-node-pods.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ spec:
2323
containers:
2424
- args:
2525
- --resources=pods
26-
- --node=""
26+
- --enable-no-node-scrape
2727
image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
2828
livenessProbe:
2929
httpGet:

‎jsonnet/kube-state-metrics/kube-state-metrics.libsonnet

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@
377377
local c = ksm.deployment.spec.template.spec.containers[0] {
378378
args: [
379379
'--resources=pods',
380-
'--node=""',
380+
'--enable-no-node-scrape',
381381
],
382382
name: shardksmname,
383383
};
@@ -410,7 +410,7 @@
410410
local c = ksm.deployment.spec.template.spec.containers[0] {
411411
args: [
412412
'--resources=pods',
413-
'--node=""',
413+
'--enable-no-node-scrape',
414414
],
415415
};
416416
local shardksmname = ksm.name + "-no-node-pods";

‎pkg/app/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
224224

225225
namespaces := opts.Namespaces.GetNamespaces()
226226
nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
227-
nodeFieldSelector := opts.Node.GetNodeFieldSelector()
227+
nodeFieldSelector := opts.Node.GetNodeFieldSelector(opts.EnableNoNodeScrape)
228228
merged, err := storeBuilder.MergeFieldSelectors([]string{nsFieldSelector, nodeFieldSelector})
229229
if err != nil {
230230
return err

‎pkg/options/options.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ type Options struct {
5959
Namespaces NamespaceList `yaml:"namespaces"`
6060
NamespacesDenylist NamespaceList `yaml:"namespaces_denylist"`
6161
Node NodeType `yaml:"node"`
62+
EnableNoNodeScrape bool `yaml:"enable_no_node_scrape"`
6263
Pod string `yaml:"pod"`
6364
Port int `yaml:"port"`
6465
Resources ResourceSet `yaml:"resources"`
@@ -90,7 +91,6 @@ func NewOptions() *Options {
9091
MetricAllowlist: MetricSet{},
9192
MetricDenylist: MetricSet{},
9293
MetricOptInList: MetricSet{},
93-
Node: NodeType{},
9494
AnnotationsAllowList: LabelsAllowList{},
9595
LabelsAllowList: LabelsAllowList{},
9696
}
@@ -138,6 +138,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
138138

139139
o.cmd.Flags().BoolVar(&o.CustomResourcesOnly, "custom-resource-state-only", false, "Only provide Custom Resource State metrics (experimental)")
140140
o.cmd.Flags().BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.")
141+
o.cmd.Flags().BoolVar(&o.EnableNoNodeScrape, "enable-no-node-scrape", false, "This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of no scheduled pods is scraped. This is experimental.")
141142
o.cmd.Flags().BoolVarP(&o.Help, "help", "h", false, "Print Help text")
142143
o.cmd.Flags().BoolVarP(&o.UseAPIServerCache, "use-apiserver-cache", "", false, "Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.")
143144
o.cmd.Flags().Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)")
@@ -156,7 +157,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
156157
o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file")
157158
o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
158159
o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file")
159-
o.cmd.Flags().Var(&o.Node, "node", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
160+
o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
160161
o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
161162
o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.")
162163
o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
@@ -186,7 +187,7 @@ func (o *Options) Usage() {
186187
// Validate validates arguments
187188
func (o *Options) Validate() error {
188189
shardableResource := "pods"
189-
if o.Node.String() == "" {
190+
if o.Node == "" {
190191
return nil
191192
}
192193
for _, x := range o.Resources.AsSlice() {

‎pkg/options/types.go

Lines changed: 7 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -105,61 +105,18 @@ func (r *ResourceSet) Type() string {
105105
}
106106

107107
// NodeType represents a nodeName to query from.
108-
type NodeType map[string]struct{}
109-
110-
// Set converts a comma-separated string of nodename into a slice and appends it to the NodeList
111-
func (n *NodeType) Set(value string) error {
112-
s := *n
113-
cols := strings.Split(value, ",")
114-
for _, col := range cols {
115-
col = strings.TrimSpace(col)
116-
if len(col) != 0 {
117-
s[col] = struct{}{}
118-
}
119-
}
120-
return nil
121-
}
122-
123-
// AsSlice returns the LabelsAllowList in the form of plain string slice.
124-
func (n NodeType) AsSlice() []string {
125-
cols := make([]string, 0, len(n))
126-
for col := range n {
127-
cols = append(cols, col)
128-
}
129-
return cols
130-
}
131-
132-
func (n NodeType) String() string {
133-
klog.InfoS("n.AsSlice()", n.AsSlice())
134-
return strings.Join(n.AsSlice(), ",")
135-
}
136-
137-
// Type returns a descriptive string about the NodeList type.
138-
func (n *NodeType) Type() string {
139-
return "string"
140-
}
108+
type NodeType string
141109

142110
// GetNodeFieldSelector returns a nodename field selector.
143-
func (n *NodeType) GetNodeFieldSelector() string {
144-
if nil == n {
145-
klog.InfoS("Using node type is nil")
146-
return EmptyFieldSelector()
147-
}
148-
nodeName := n.String()
149-
// `--node=""` find pods without node name assigned which uses fieldselector spec.nodeName=""
150-
klog.InfoS("Using node name", nodeName)
151-
if nodeName == "" {
111+
func (n *NodeType) GetNodeFieldSelector(noNodeAssigned bool) string {
112+
if noNodeAssigned {
152113
klog.InfoS("Using spec.nodeName= to select unscheduable pods without node")
153114
return "spec.nodeName="
154115
}
155-
klog.InfoS("Using spec.nodeName=", nodeName)
156-
return fields.OneTermEqualSelector("spec.nodeName", nodeName).String()
157-
158-
}
159-
160-
// NodeValue represents a nodeName to query from.
161-
type NodeValue interface {
162-
GetNodeFieldSelector() string
116+
if string(*n) != "" {
117+
return fields.OneTermEqualSelector("spec.nodeName", string(*n)).String()
118+
}
119+
return EmptyFieldSelector()
163120
}
164121

165122
// EmptyFieldSelector returns an empty field selector.

‎pkg/options/types_test.go

Lines changed: 35 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -162,37 +162,38 @@ func TestNodeFieldSelector(t *testing.T) {
162162
Wanted string
163163
}{
164164
{
165-
Desc: "with node name",
165+
Desc: "empty node name",
166+
Node: "",
166167
Wanted: "",
167168
},
168169
{
169170
Desc: "with node name",
170-
Node: nil,
171-
Wanted: "",
172-
},
173-
{
174-
Desc: "empty node name",
175-
Node: NodeType(
176-
map[string]struct{}{
177-
"": {},
178-
},
179-
),
180-
Wanted: "spec.nodeName=",
181-
},
182-
{
183-
Desc: "with node name",
184-
Node: NodeType(
185-
map[string]struct{}{
186-
"k8s-node-1": {},
187-
},
188-
),
171+
Node: "k8s-node-1",
189172
Wanted: "spec.nodeName=k8s-node-1",
190173
},
191174
}
192175

193176
for _, test := range tests {
194177
node := test.Node
195-
actual := node.GetNodeFieldSelector()
178+
actual := node.GetNodeFieldSelector(false)
179+
if !reflect.DeepEqual(actual, test.Wanted) {
180+
t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual)
181+
}
182+
}
183+
tests1 := []struct {
184+
Desc string
185+
Node NodeType
186+
Wanted string
187+
}{
188+
{
189+
Desc: "empty node name",
190+
Node: "",
191+
Wanted: "spec.nodeName=",
192+
},
193+
}
194+
for _, test := range tests1 {
195+
node := test.Node
196+
actual := node.GetNodeFieldSelector(true)
196197
if !reflect.DeepEqual(actual, test.Wanted) {
197198
t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual)
198199
}
@@ -211,75 +212,51 @@ func TestMergeFieldSelectors(t *testing.T) {
211212
Desc: "empty DeniedNamespaces",
212213
Namespaces: NamespaceList{"default", "kube-system"},
213214
DeniedNamespaces: NamespaceList{},
214-
Node: NodeType(
215-
map[string]struct{}{
216-
"": {},
217-
},
218-
),
219-
Wanted: "spec.nodeName=",
215+
Node: "",
216+
Wanted: "",
220217
},
221218
{
222219
Desc: "all DeniedNamespaces",
223220
Namespaces: DefaultNamespaces,
224221
DeniedNamespaces: NamespaceList{"some-system"},
225-
Node: NodeType(
226-
map[string]struct{}{
227-
"": {},
228-
},
229-
),
230-
Wanted: "metadata.namespace!=some-system,spec.nodeName=",
222+
Node: "",
223+
Wanted: "metadata.namespace!=some-system",
231224
},
232225
{
233226
Desc: "general case",
234227
Namespaces: DefaultNamespaces,
235228
DeniedNamespaces: NamespaceList{"case1-system", "case2-system"},
236-
Node: NodeType(
237-
map[string]struct{}{
238-
"": {},
239-
},
240-
),
241-
Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=",
229+
Node: "",
230+
Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system",
242231
},
243232
{
244233
Desc: "empty DeniedNamespaces",
245234
Namespaces: NamespaceList{"default", "kube-system"},
246235
DeniedNamespaces: NamespaceList{},
247-
Node: NodeType(
248-
map[string]struct{}{
249-
"k8s-node-1": {},
250-
},
251-
),
252-
Wanted: "spec.nodeName=k8s-node-1",
236+
Node: "k8s-node-1",
237+
Wanted: "spec.nodeName=k8s-node-1",
253238
},
254239
{
255240
Desc: "all DeniedNamespaces",
256241
Namespaces: DefaultNamespaces,
257242
DeniedNamespaces: NamespaceList{"some-system"},
258-
Node: NodeType(
259-
map[string]struct{}{
260-
"k8s-node-1": {},
261-
},
262-
),
263-
Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1",
243+
Node: "k8s-node-1",
244+
Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1",
264245
},
265246
{
266247
Desc: "general case",
267248
Namespaces: DefaultNamespaces,
268249
DeniedNamespaces: NamespaceList{"case1-system", "case2-system"},
269-
Node: NodeType(
270-
map[string]struct{}{
271-
"k8s-node-1": {},
272-
},
273-
),
274-
Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1",
250+
Node: "k8s-node-1",
251+
Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1",
275252
},
276253
}
277254

278255
for _, test := range tests {
279256
ns := test.Namespaces
280257
deniedNS := test.DeniedNamespaces
281258
selector1 := ns.GetExcludeNSFieldSelector(deniedNS)
282-
selector2 := test.Node.GetNodeFieldSelector()
259+
selector2 := test.Node.GetNodeFieldSelector(false)
283260
actual, err := MergeFieldSelectors([]string{selector1, selector2})
284261
if err != nil {
285262
t.Errorf("Test error for Desc: %s. Can't merge field selector %v.", test.Desc, err)

‎tests/e2e.sh

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,7 @@ function test_daemonset() {
124124
sed -i "s|${KUBE_STATE_METRICS_CURRENT_IMAGE_NAME}:v.*|${KUBE_STATE_METRICS_IMAGE_NAME}:${KUBE_STATE_METRICS_IMAGE_TAG}|g" ./examples/daemonsetsharding/daemonset.yaml
125125
sed -i "s|${KUBE_STATE_METRICS_CURRENT_IMAGE_NAME}:v.*|${KUBE_STATE_METRICS_IMAGE_NAME}:${KUBE_STATE_METRICS_IMAGE_TAG}|g" ./examples/daemonsetsharding/deployment-no-node-pods.yaml
126126

127-
cat ./examples/daemonsetsharding/deployment-no-node-pods.yaml
128-
sleep 3
129127
kubectl get deployment -n kube-system
130-
ls ./examples/daemonsetsharding
131128
kubectl create -f ./examples/daemonsetsharding
132129
kube_state_metrics_up kube-state-metrics-no-node-pods
133130
kube_state_metrics_up kube-state-metrics
@@ -137,12 +134,6 @@ function test_daemonset() {
137134
kube_pod_up pendingpod2
138135

139136
kubectl get deployment -n default
140-
# curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-shard:http-metrics/proxy/metrics" >${KUBE_STATE_METRICS_LOG_DIR}/daemonset-scraped-metrics
141-
# curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy/metrics" >${KUBE_STATE_METRICS_LOG_DIR}/deployment-scraped-metrics
142-
# curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-no-node-pods:http-metrics/proxy/metrics" >${KUBE_STATE_METRICS_LOG_DIR}/deployment-scraped-no-node-metrics
143-
144-
# cat ${KUBE_STATE_METRICS_LOG_DIR}/daemonset-scraped-metrics ${KUBE_STATE_METRICS_LOG_DIR}/deployment-scraped-metrics ${KUBE_STATE_METRICS_LOG_DIR}/deployment-scraped-no-node-metrics >> ${KUBE_STATE_METRICS_LOG_DIR}/all-metrics
145-
# cat ${KUBE_STATE_METRICS_LOG_DIR}/all-metrics | grep "kube_pod_info"
146137
runningpod1="$(curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-shard:http-metrics/proxy/metrics" | grep "runningpod1" | grep -c "kube_pod_info" )"
147138
node1="$(curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy/metrics" | grep -c "# TYPE kube_node_info" )"
148139
expected_num_pod=1
@@ -157,11 +148,9 @@ function test_daemonset() {
157148
fi
158149

159150
kubectl logs deployment/kube-state-metrics-no-node-pods -n kube-system
160-
sleep 3
161-
curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-no-node-pods:http-metrics/proxy/metrics"
162-
sleep 3
151+
sleep 2
163152
kubectl get pods -A --field-selector spec.nodeName=""
164-
sleep 3
153+
sleep 2
165154
pendingpod2="$(curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-no-node-pods:http-metrics/proxy/metrics" | grep "pendingpod2" | grep -c "kube_pod_info" )"
166155
if [ "${pendingpod2}" != "${expected_num_pod}" ]; then
167156
echo "metric kube_pod_info for pendingpod2 doesn't show up only once, got ${runningpod1} times"
@@ -244,6 +233,8 @@ echo "kube-state-metrics is up and running"
244233
echo "start e2e test for kube-state-metrics"
245234
KSM_HTTP_METRICS_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy'
246235
KSM_TELEMETRY_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:telemetry/proxy'
236+
237+
kubectl --namespace=kube-system logs deployment/kube-state-metrics kube-state-metrics
247238
go test -v ./tests/e2e/main_test.go --ksm-http-metrics-url=${KSM_HTTP_METRICS_URL} --ksm-telemetry-url=${KSM_TELEMETRY_URL}
248239

249240
# TODO: re-implement the following test cases in Go with the goal of removing this file.

0 commit comments

Comments
 (0)