Skip to content

Commit 624da8b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#88110 from fromanirh/refactor-get-current-kubelet-conf
e2e: e2e_node: refactor getCurrentKubeletConfig
2 parents 4e79344 + 53cda47 commit 624da8b

File tree

6 files changed

+143
-170
lines changed

6 files changed

+143
-170
lines changed

‎test/e2e/framework/kubelet/BUILD‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"config.go",
67
"kubelet_pods.go",
78
"stats.go",
89
],
910
importpath = "k8s.io/kubernetes/test/e2e/framework/kubelet",
1011
visibility = ["//visibility:public"],
1112
deps = [
13+
"//pkg/kubelet/apis/config:go_default_library",
1214
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
1315
"//pkg/kubelet/dockershim/metrics:go_default_library",
1416
"//pkg/master/ports:go_default_library",
@@ -17,8 +19,11 @@ go_library(
1719
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
1820
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
1921
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
22+
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
2023
"//staging/src/k8s.io/client-go/rest:go_default_library",
24+
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
2125
"//test/e2e/framework:go_default_library",
26+
"//test/e2e/framework/kubectl:go_default_library",
2227
"//test/e2e/framework/metrics:go_default_library",
2328
],
2429
)

‎test/e2e/framework/kubelet/config.go‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kubelet
18+
19+
import (
20+
"crypto/tls"
21+
"encoding/json"
22+
"fmt"
23+
"io/ioutil"
24+
"net/http"
25+
"regexp"
26+
"strconv"
27+
"time"
28+
29+
"k8s.io/apimachinery/pkg/util/wait"
30+
"k8s.io/client-go/kubernetes/scheme"
31+
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
32+
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
33+
34+
"k8s.io/kubernetes/test/e2e/framework"
35+
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
36+
)
37+
38+
// GetCurrentKubeletConfig fetches the current Kubelet Config for the given node
39+
func GetCurrentKubeletConfig(nodeName, namespace string, useProxy bool) (*kubeletconfig.KubeletConfiguration, error) {
40+
resp := pollConfigz(5*time.Minute, 5*time.Second, nodeName, namespace, useProxy)
41+
if resp == nil {
42+
return nil, fmt.Errorf("failed to fetch /configz from %q", nodeName)
43+
}
44+
kubeCfg, err := decodeConfigz(resp)
45+
if err != nil {
46+
return nil, err
47+
}
48+
return kubeCfg, nil
49+
}
50+
51+
// returns a status 200 response from the /configz endpoint or nil if fails
52+
func pollConfigz(timeout time.Duration, pollInterval time.Duration, nodeName, namespace string, useProxy bool) *http.Response {
53+
endpoint := ""
54+
if useProxy {
55+
// start local proxy, so we can send graceful deletion over query string, rather than body parameter
56+
framework.Logf("Opening proxy to cluster")
57+
tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, namespace)
58+
cmd := tk.KubectlCmd("proxy", "-p", "0")
59+
stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd)
60+
framework.ExpectNoError(err)
61+
defer stdout.Close()
62+
defer stderr.Close()
63+
defer framework.TryKill(cmd)
64+
65+
buf := make([]byte, 128)
66+
var n int
67+
n, err = stdout.Read(buf)
68+
framework.ExpectNoError(err)
69+
output := string(buf[:n])
70+
proxyRegexp := regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)")
71+
match := proxyRegexp.FindStringSubmatch(output)
72+
framework.ExpectEqual(len(match), 2)
73+
port, err := strconv.Atoi(match[1])
74+
framework.ExpectNoError(err)
75+
framework.Logf("http requesting node kubelet /configz")
76+
endpoint = fmt.Sprintf("http://127.0.0.1:%d/api/v1/nodes/%s/proxy/configz", port, nodeName)
77+
} else {
78+
endpoint = fmt.Sprintf("http://127.0.0.1:8080/api/v1/nodes/%s/proxy/configz", framework.TestContext.NodeName)
79+
}
80+
tr := &http.Transport{
81+
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
82+
}
83+
client := &http.Client{Transport: tr}
84+
req, err := http.NewRequest("GET", endpoint, nil)
85+
framework.ExpectNoError(err)
86+
req.Header.Add("Accept", "application/json")
87+
88+
var resp *http.Response
89+
wait.PollImmediate(pollInterval, timeout, func() (bool, error) {
90+
resp, err = client.Do(req)
91+
if err != nil {
92+
framework.Logf("Failed to get /configz, retrying. Error: %v", err)
93+
return false, nil
94+
}
95+
if resp.StatusCode != 200 {
96+
framework.Logf("/configz response status not 200, retrying. Response was: %+v", resp)
97+
return false, nil
98+
}
99+
100+
return true, nil
101+
})
102+
return resp
103+
}
104+
105+
// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type).
106+
func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) {
107+
// This hack because /configz reports the following structure:
108+
// {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}}
109+
type configzWrapper struct {
110+
ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"`
111+
}
112+
113+
configz := configzWrapper{}
114+
kubeCfg := kubeletconfig.KubeletConfiguration{}
115+
116+
contentsBytes, err := ioutil.ReadAll(resp.Body)
117+
if err != nil {
118+
return nil, err
119+
}
120+
121+
err = json.Unmarshal(contentsBytes, &configz)
122+
if err != nil {
123+
return nil, err
124+
}
125+
126+
err = scheme.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
127+
if err != nil {
128+
return nil, err
129+
}
130+
131+
return &kubeCfg, nil
132+
}

‎test/e2e/windows/BUILD‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ go_library(
2121
importpath = "k8s.io/kubernetes/test/e2e/windows",
2222
visibility = ["//visibility:public"],
2323
deps = [
24-
"//pkg/kubelet/apis/config:go_default_library",
2524
"//staging/src/k8s.io/api/core/v1:go_default_library",
2625
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
2726
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@@ -31,11 +30,8 @@ go_library(
3130
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
3231
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
3332
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
34-
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
3533
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
36-
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
3734
"//test/e2e/framework:go_default_library",
38-
"//test/e2e/framework/kubectl:go_default_library",
3935
"//test/e2e/framework/kubelet:go_default_library",
4036
"//test/e2e/framework/metrics:go_default_library",
4137
"//test/e2e/framework/node:go_default_library",

‎test/e2e/windows/memory_limits.go‎

Lines changed: 2 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,7 @@ package windows
1818

1919
import (
2020
"context"
21-
"crypto/tls"
22-
"encoding/json"
2321
"fmt"
24-
"io/ioutil"
25-
"net/http"
26-
"regexp"
2722
"strconv"
2823
"time"
2924

@@ -32,11 +27,8 @@ import (
3227
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3328
"k8s.io/apimachinery/pkg/labels"
3429
"k8s.io/apimachinery/pkg/util/uuid"
35-
"k8s.io/client-go/kubernetes/scheme"
36-
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
37-
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
3830
"k8s.io/kubernetes/test/e2e/framework"
39-
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
31+
e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
4032
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
4133
imageutils "k8s.io/kubernetes/test/utils/image"
4234

@@ -197,7 +189,7 @@ func getNodeMemory(f *framework.Framework) nodeMemory {
197189

198190
nodeName := nodeList.Items[0].ObjectMeta.Name
199191

200-
kubeletConfig, err := getCurrentKubeletConfig(nodeName, f.Namespace.Name)
192+
kubeletConfig, err := e2ekubelet.GetCurrentKubeletConfig(nodeName, f.Namespace.Name, true)
201193
framework.ExpectNoError(err)
202194

203195
systemReserve, err := resource.ParseQuantity(kubeletConfig.SystemReserved["memory"])
@@ -250,91 +242,3 @@ func getTotalAllocatableMemory(f *framework.Framework) *resource.Quantity {
250242

251243
return totalAllocatable
252244
}
253-
254-
// getCurrentKubeletConfig modified from test/e2e_node/util.go
255-
func getCurrentKubeletConfig(nodeName, namespace string) (*kubeletconfig.KubeletConfiguration, error) {
256-
257-
resp := pollConfigz(5*time.Minute, 5*time.Second, nodeName, namespace)
258-
kubeCfg, err := decodeConfigz(resp)
259-
if err != nil {
260-
return nil, err
261-
}
262-
return kubeCfg, nil
263-
}
264-
265-
// Causes the test to fail, or returns a status 200 response from the /configz endpoint
266-
func pollConfigz(timeout time.Duration, pollInterval time.Duration, nodeName, namespace string) *http.Response {
267-
// start local proxy, so we can send graceful deletion over query string, rather than body parameter
268-
ginkgo.By("Opening proxy to cluster")
269-
tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, namespace)
270-
cmd := tk.KubectlCmd("proxy", "-p", "0")
271-
stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd)
272-
framework.ExpectNoError(err)
273-
defer stdout.Close()
274-
defer stderr.Close()
275-
defer framework.TryKill(cmd)
276-
buf := make([]byte, 128)
277-
var n int
278-
n, err = stdout.Read(buf)
279-
framework.ExpectNoError(err)
280-
output := string(buf[:n])
281-
proxyRegexp := regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)")
282-
match := proxyRegexp.FindStringSubmatch(output)
283-
framework.ExpectEqual(len(match), 2)
284-
port, err := strconv.Atoi(match[1])
285-
framework.ExpectNoError(err)
286-
ginkgo.By("http requesting node kubelet /configz")
287-
endpoint := fmt.Sprintf("http://127.0.0.1:%d/api/v1/nodes/%s/proxy/configz", port, nodeName)
288-
tr := &http.Transport{
289-
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
290-
}
291-
client := &http.Client{Transport: tr}
292-
req, err := http.NewRequest("GET", endpoint, nil)
293-
framework.ExpectNoError(err)
294-
req.Header.Add("Accept", "application/json")
295-
296-
var resp *http.Response
297-
gomega.Eventually(func() bool {
298-
resp, err = client.Do(req)
299-
if err != nil {
300-
framework.Logf("Failed to get /configz, retrying. Error: %v", err)
301-
return false
302-
}
303-
if resp.StatusCode != 200 {
304-
framework.Logf("/configz response status not 200, retrying. Response was: %+v", resp)
305-
return false
306-
}
307-
308-
return true
309-
}, timeout, pollInterval).Should(gomega.Equal(true))
310-
return resp
311-
}
312-
313-
// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type).
314-
func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) {
315-
// This hack because /configz reports the following structure:
316-
// {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}}
317-
type configzWrapper struct {
318-
ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"`
319-
}
320-
321-
configz := configzWrapper{}
322-
kubeCfg := kubeletconfig.KubeletConfiguration{}
323-
324-
contentsBytes, err := ioutil.ReadAll(resp.Body)
325-
if err != nil {
326-
return nil, err
327-
}
328-
329-
err = json.Unmarshal(contentsBytes, &configz)
330-
if err != nil {
331-
return nil, err
332-
}
333-
334-
err = scheme.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
335-
if err != nil {
336-
return nil, err
337-
}
338-
339-
return &kubeCfg, nil
340-
}

‎test/e2e_node/BUILD‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ go_library(
2727
deps = [
2828
"//pkg/features:go_default_library",
2929
"//pkg/kubelet/apis/config:go_default_library",
30-
"//pkg/kubelet/apis/config/scheme:go_default_library",
3130
"//pkg/kubelet/apis/podresources:go_default_library",
3231
"//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
3332
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
@@ -50,6 +49,7 @@ go_library(
5049
"//test/e2e/common:go_default_library",
5150
"//test/e2e/framework:go_default_library",
5251
"//test/e2e/framework/gpu:go_default_library",
52+
"//test/e2e/framework/kubelet:go_default_library",
5353
"//test/e2e/framework/metrics:go_default_library",
5454
"//test/e2e/framework/node:go_default_library",
5555
"//test/e2e/framework/testfiles:go_default_library",
@@ -69,7 +69,6 @@ go_library(
6969
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
7070
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
7171
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
72-
"//test/e2e/framework/kubelet:go_default_library",
7372
"//test/e2e/framework/perf:go_default_library",
7473
"//test/e2e/framework/pod:go_default_library",
7574
"//test/e2e/perftype:go_default_library",
@@ -89,7 +88,6 @@ go_library(
8988
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
9089
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
9190
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
92-
"//test/e2e/framework/kubelet:go_default_library",
9391
"//test/e2e/framework/perf:go_default_library",
9492
"//test/e2e/framework/pod:go_default_library",
9593
"//test/e2e/perftype:go_default_library",

0 commit comments

Comments
 (0)