Skip to content

Commit f310ac0

Browse files
neolit123neolit123
committed
kubeadm: graduate WaitForAllControlPlaneComponents to Beta
- Set the feature gate to Beta and enabled by default. - Make sure that the source of truth for which address/port to use for a component health check comes from the respective component static Pod manifest. That is done to comply with any user --patches that are applied on top of the ClusterConfiguration.
1 parent 3bc8f01 commit f310ac0

File tree

9 files changed

+409
-90
lines changed

9 files changed

+409
-90
lines changed

‎cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,17 @@ import (
2525
"github.com/lithammer/dedent"
2626
"github.com/pkg/errors"
2727

28+
v1 "k8s.io/api/core/v1"
2829
clientset "k8s.io/client-go/kubernetes"
2930
kubeletconfig "k8s.io/kubelet/config/v1beta1"
3031

3132
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
3233
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
34+
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
3335
"k8s.io/kubernetes/cmd/kubeadm/app/features"
3436
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
3537
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
38+
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
3639
)
3740

3841
var (
@@ -122,10 +125,15 @@ func runWaitControlPlanePhase(c workflow.RunData) error {
122125
return handleError(err)
123126
}
124127

128+
var podMap map[string]*v1.Pod
125129
waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration)
126130
if features.Enabled(data.Cfg().ClusterConfiguration.FeatureGates, features.WaitForAllControlPlaneComponents) {
127-
err = waiter.WaitForControlPlaneComponents(&data.Cfg().ClusterConfiguration,
128-
data.Cfg().LocalAPIEndpoint.AdvertiseAddress)
131+
podMap, err = staticpodutil.ReadMultipleStaticPodsFromDisk(data.ManifestDir(),
132+
constants.ControlPlaneComponents...)
133+
if err == nil {
134+
err = waiter.WaitForControlPlaneComponents(podMap,
135+
data.Cfg().LocalAPIEndpoint.AdvertiseAddress)
136+
}
129137
} else {
130138
err = waiter.WaitForAPI()
131139
}

‎cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@ import (
2626
"k8s.io/klog/v2"
2727

2828
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
29+
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
2930
"k8s.io/kubernetes/cmd/kubeadm/app/features"
3031
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
3132
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
33+
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
3234
)
3335

3436
// NewWaitControlPlanePhase is a hidden phase that runs after the control-plane and etcd phases
@@ -71,7 +73,12 @@ func runWaitControlPlanePhase(c workflow.RunData) error {
7173
}
7274

7375
waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration)
74-
if err := waiter.WaitForControlPlaneComponents(&initCfg.ClusterConfiguration,
76+
pods, err := staticpodutil.ReadMultipleStaticPodsFromDisk(data.ManifestDir(),
77+
constants.ControlPlaneComponents...)
78+
if err != nil {
79+
return err
80+
}
81+
if err = waiter.WaitForControlPlaneComponents(pods,
7582
data.Cfg().ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil {
7683
return err
7784
}

‎cmd/kubeadm/app/features/features.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ var InitFeatureGates = FeatureList{
5353
DeprecationMessage: "Deprecated in favor of the core kubelet feature UserNamespacesSupport which is beta since 1.30." +
5454
" Once UserNamespacesSupport graduates to GA, kubeadm will start using it and RootlessControlPlane will be removed.",
5555
},
56-
WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
56+
WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
5757
ControlPlaneKubeletLocalMode: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
5858
NodeLocalCRISocket: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
5959
}

‎cmd/kubeadm/app/phases/upgrade/staticpods_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/pkg/errors"
3131
"go.etcd.io/etcd/client/pkg/v3/transport"
3232

33+
v1 "k8s.io/api/core/v1"
3334
"k8s.io/client-go/tools/clientcmd"
3435
certutil "k8s.io/client-go/util/cert"
3536

@@ -99,7 +100,7 @@ func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter {
99100
}
100101

101102
// WaitForControlPlaneComponents just returns a dummy nil, to indicate that the program should just proceed
102-
func (w *fakeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error {
103+
func (w *fakeWaiter) WaitForControlPlaneComponents(podsMap map[string]*v1.Pod, apiServerAddress string) error {
103104
return nil
104105
}
105106

‎cmd/kubeadm/app/util/apiclient/wait.go‎

Lines changed: 133 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"io"
2424
"net"
2525
"net/http"
26+
"strings"
2627
"time"
2728

2829
"github.com/pkg/errors"
@@ -34,14 +35,27 @@ import (
3435
"k8s.io/apimachinery/pkg/util/wait"
3536
clientset "k8s.io/client-go/kubernetes"
3637

37-
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
3838
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
3939
)
4040

41+
const (
42+
// TODO: switch to /livez once all components support it
43+
// and delete the endpointHealthz constant.
44+
// https://github.com/kubernetes/kubernetes/issues/118158
45+
endpointHealthz = "healthz"
46+
endpointLivez = "livez"
47+
48+
argPort = "secure-port"
49+
argBindAddress = "bind-address"
50+
// By default, for kube-api-server, kubeadm does not apply a --bind-address flag.
51+
// Check --advertise-address instead.
52+
argAdvertiseAddress = "advertise-address"
53+
)
54+
4155
// Waiter is an interface for waiting for criteria in Kubernetes to happen
4256
type Waiter interface {
4357
// WaitForControlPlaneComponents waits for all control plane components to be ready.
44-
WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error
58+
WaitForControlPlaneComponents(podMap map[string]*v1.Pod, apiServerAddress string) error
4559
// WaitForAPI waits for the API Server's /healthz endpoint to become "ok"
4660
// TODO: remove WaitForAPI once WaitForAllControlPlaneComponents goes GA:
4761
// https://github.com/kubernetes/kubeadm/issues/2907
@@ -77,80 +91,147 @@ func NewKubeWaiter(client clientset.Interface, timeout time.Duration, writer io.
7791
}
7892
}
7993

94+
// controlPlaneComponent holds a component name and an URL
95+
// on which to perform health checks.
8096
type controlPlaneComponent struct {
8197
name string
8298
url string
8399
}
84100

85-
const (
86-
// TODO: switch to /livez once all components support it
87-
// and delete the endpointHealthz constant.
88-
// https://github.com/kubernetes/kubernetes/issues/118158
89-
endpointHealthz = "healthz"
90-
endpointLivez = "livez"
91-
)
92-
93-
// getControlPlaneComponents takes a ClusterConfiguration and returns a slice of
94-
// control plane components and their health check URLs.
95-
func getControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, defaultAddressAPIServer string) []controlPlaneComponent {
96-
const (
97-
portArg = "secure-port"
98-
bindAddressArg = "bind-address"
99-
// By default, for kube-api-server, kubeadm does not apply a --bind-address flag.
100-
// Check --advertise-address instead, which can override the defaultAddressAPIServer value.
101-
advertiseAddressArg = "advertise-address"
102-
// By default kubeadm deploys the kube-controller-manager and kube-scheduler
103-
// with --bind-address=127.0.0.1. This should match get{Scheduler|ControllerManager}Command().
104-
defaultAddressKCM = "127.0.0.1"
105-
defaultAddressScheduler = "127.0.0.1"
101+
// getControlPlaneComponentAddressAndPort parses the command in a static Pod
102+
// container and extracts the values of the given args.
103+
func getControlPlaneComponentAddressAndPort(pod *v1.Pod, name string, args []string) ([]string, error) {
104+
var (
105+
values = make([]string, len(args))
106+
container *v1.Container
106107
)
107108

108-
portAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, portArg, -1)
109-
if idx == -1 {
110-
portAPIServer = fmt.Sprintf("%d", constants.KubeAPIServerPort)
109+
if pod == nil {
110+
return values, errors.Errorf("got nil Pod for component %q", name)
111111
}
112-
portKCM, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, portArg, -1)
113-
if idx == -1 {
114-
portKCM = fmt.Sprintf("%d", constants.KubeControllerManagerPort)
112+
113+
for i, c := range pod.Spec.Containers {
114+
if len(c.Command) == 0 {
115+
continue
116+
}
117+
if c.Command[0] == name {
118+
container = &pod.Spec.Containers[i]
119+
break
120+
}
115121
}
116-
portScheduler, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, portArg, -1)
117-
if idx == -1 {
118-
portScheduler = fmt.Sprintf("%d", constants.KubeSchedulerPort)
122+
if container == nil {
123+
return values, errors.Errorf("the Pod has no container command starting with %q", name)
119124
}
120125

121-
addressAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, advertiseAddressArg, -1)
122-
if idx == -1 {
123-
addressAPIServer = defaultAddressAPIServer
126+
for _, line := range container.Command {
127+
for i, arg := range args {
128+
line = strings.TrimSpace(line)
129+
if !strings.HasPrefix(line, "--"+arg) && !strings.HasPrefix(line, "-"+arg) {
130+
continue
131+
}
132+
_, value, found := strings.Cut(line, "=")
133+
if !found {
134+
_, value, _ = strings.Cut(line, " ")
135+
}
136+
values[i] = value
137+
}
124138
}
125-
addressKCM, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, bindAddressArg, -1)
126-
if idx == -1 {
127-
addressKCM = defaultAddressKCM
139+
return values, nil
140+
}
141+
142+
// getControlPlaneComponents reads the static Pods of control plane components
143+
// and returns a slice of 'controlPlaneComponent'.
144+
func getControlPlaneComponents(podMap map[string]*v1.Pod, addressAPIServer string) ([]controlPlaneComponent, error) {
145+
var (
146+
// By default kubeadm deploys the kube-controller-manager and kube-scheduler
147+
// with --bind-address=127.0.0.1. This should match get{Scheduler|ControllerManager}Command().
148+
addressKCM = "127.0.0.1"
149+
addressScheduler = "127.0.0.1"
150+
151+
portAPIServer = fmt.Sprintf("%d", constants.KubeAPIServerPort)
152+
portKCM = fmt.Sprintf("%d", constants.KubeControllerManagerPort)
153+
portScheduler = fmt.Sprintf("%d", constants.KubeSchedulerPort)
154+
155+
errs []error
156+
result []controlPlaneComponent
157+
)
158+
159+
type componentConfig struct {
160+
name string
161+
podKey string
162+
args []string
163+
defaultAddr string
164+
defaultPort string
165+
endpoint string
128166
}
129-
addressScheduler, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, bindAddressArg, -1)
130-
if idx == -1 {
131-
addressScheduler = defaultAddressScheduler
167+
168+
components := []componentConfig{
169+
{
170+
name: "kube-apiserver",
171+
podKey: constants.KubeAPIServer,
172+
args: []string{argAdvertiseAddress, argPort},
173+
defaultAddr: addressAPIServer,
174+
defaultPort: portAPIServer,
175+
endpoint: endpointLivez,
176+
},
177+
{
178+
name: "kube-controller-manager",
179+
podKey: constants.KubeControllerManager,
180+
args: []string{argBindAddress, argPort},
181+
defaultAddr: addressKCM,
182+
defaultPort: portKCM,
183+
endpoint: endpointHealthz,
184+
},
185+
{
186+
name: "kube-scheduler",
187+
podKey: constants.KubeScheduler,
188+
args: []string{argBindAddress, argPort},
189+
defaultAddr: addressScheduler,
190+
defaultPort: portScheduler,
191+
endpoint: endpointLivez,
192+
},
132193
}
133194

134-
getURL := func(address, port, endpoint string) string {
135-
return fmt.Sprintf(
136-
"https://%s/%s",
137-
net.JoinHostPort(address, port),
138-
endpoint,
195+
for _, component := range components {
196+
address, port := component.defaultAddr, component.defaultPort
197+
198+
values, err := getControlPlaneComponentAddressAndPort(
199+
podMap[component.podKey],
200+
component.podKey,
201+
component.args,
139202
)
203+
if err != nil {
204+
errs = append(errs, err)
205+
}
206+
207+
if len(values[0]) != 0 {
208+
address = values[0]
209+
}
210+
if len(values[1]) != 0 {
211+
port = values[1]
212+
}
213+
214+
result = append(result, controlPlaneComponent{
215+
name: component.name,
216+
url: fmt.Sprintf("https://%s/%s", net.JoinHostPort(address, port), component.endpoint),
217+
})
140218
}
141-
return []controlPlaneComponent{
142-
{name: "kube-apiserver", url: getURL(addressAPIServer, portAPIServer, endpointLivez)},
143-
{name: "kube-controller-manager", url: getURL(addressKCM, portKCM, endpointHealthz)},
144-
{name: "kube-scheduler", url: getURL(addressScheduler, portScheduler, endpointLivez)},
219+
220+
if len(errs) > 0 {
221+
return nil, utilerrors.NewAggregate(errs)
145222
}
223+
return result, nil
146224
}
147225

148226
// WaitForControlPlaneComponents waits for all control plane components to report "ok".
149-
func (w *KubeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiSeverAddress string) error {
227+
func (w *KubeWaiter) WaitForControlPlaneComponents(podMap map[string]*v1.Pod, apiSeverAddress string) error {
150228
fmt.Printf("[control-plane-check] Waiting for healthy control plane components."+
151229
" This can take up to %v\n", w.timeout)
152230

153-
components := getControlPlaneComponents(cfg, apiSeverAddress)
231+
components, err := getControlPlaneComponents(podMap, apiSeverAddress)
232+
if err != nil {
233+
return errors.Wrap(err, "could not parse the address and port of all control plane components")
234+
}
154235

155236
var errs []error
156237
errChan := make(chan error, len(components))

0 commit comments

Comments
 (0)