Skip to content

Commit 6dc11c1

Browse files
neolit123neolit123
committed
kubeadm: ensure the kubelet and kube-apiserver wait checks go first
The addition of the "super-admin.conf" functionality required init.go's Client() to create RBAC rules on its first creation. However this created a problem with the "wait-control-plane" phase of "kubeadm init" where a client is needed to connect to the API server Discovery API's "/healthz" endpoint. The logic that ensures the RBAC became the step where the API server wait was polled for. To avoid this, introduce a new InitData function ClientWithoutBootstrap. In "wait-control-plane" use this client, which has no permissions (anonymous), but is sufficient to connect to the "/healthz". Pending changes here would be: - Stop using the "/healthz", instead a regular REST client from the kubelet cert/key can be constructed. - Make the wait for kubelet / API server linear (not in go routines).
1 parent 24e6b03 commit 6dc11c1

File tree

4 files changed

+59
-27
lines changed

4 files changed

+59
-27
lines changed

‎cmd/kubeadm/app/cmd/init.go‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -496,21 +496,28 @@ func (d *initData) OutputWriter() io.Writer {
496496
return d.outputWriter
497497
}
498498

499+
// getDryRunClient creates a fake client that answers some GET calls in order to be able to do the full init flow in dry-run mode.
500+
func getDryRunClient(d *initData) (clientset.Interface, error) {
501+
svcSubnetCIDR, err := kubeadmconstants.GetKubernetesServiceCIDR(d.cfg.Networking.ServiceSubnet)
502+
if err != nil {
503+
return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", d.cfg.Networking.ServiceSubnet)
504+
}
505+
dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String())
506+
return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil
507+
}
508+
499509
// Client returns a Kubernetes client to be used by kubeadm.
500510
// This function is implemented as a singleton, thus avoiding to recreate the client when it is used by different phases.
501511
// Important. This function must be called after the admin.conf kubeconfig file is created.
502512
func (d *initData) Client() (clientset.Interface, error) {
513+
var err error
503514
if d.client == nil {
504515
if d.dryRun {
505-
svcSubnetCIDR, err := kubeadmconstants.GetKubernetesServiceCIDR(d.cfg.Networking.ServiceSubnet)
516+
d.client, err = getDryRunClient(d)
506517
if err != nil {
507-
return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", d.cfg.Networking.ServiceSubnet)
518+
return nil, err
508519
}
509-
// If we're dry-running, we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests
510-
dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String())
511-
d.client = apiclient.NewDryRunClient(dryRunGetter, os.Stdout)
512520
} else { // Use a real client
513-
var err error
514521
if !d.adminKubeConfigBootstrapped {
515522
// Call EnsureAdminClusterRoleBinding() to obtain a working client from admin.conf.
516523
d.client, err = kubeconfigphase.EnsureAdminClusterRoleBinding(kubeadmconstants.KubernetesDir, nil)
@@ -531,6 +538,28 @@ func (d *initData) Client() (clientset.Interface, error) {
531538
return d.client, nil
532539
}
533540

541+
// ClientWithoutBootstrap returns a dry-run client or a regular client from admin.conf.
542+
// Unlike Client(), it does not call EnsureAdminClusterRoleBinding() or sets d.client.
543+
// This means the client only has anonymous permissions and does not persist in initData.
544+
func (d *initData) ClientWithoutBootstrap() (clientset.Interface, error) {
545+
var (
546+
client clientset.Interface
547+
err error
548+
)
549+
if d.dryRun {
550+
client, err = getDryRunClient(d)
551+
if err != nil {
552+
return nil, err
553+
}
554+
} else { // Use a real client
555+
client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath())
556+
if err != nil {
557+
return nil, err
558+
}
559+
}
560+
return client, nil
561+
}
562+
534563
// Tokens returns an array of token strings.
535564
func (d *initData) Tokens() []string {
536565
tokens := []string{}

‎cmd/kubeadm/app/cmd/phases/init/data.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ type InitData interface {
4545
ExternalCA() bool
4646
OutputWriter() io.Writer
4747
Client() (clientset.Interface, error)
48+
ClientWithoutBootstrap() (clientset.Interface, error)
4849
Tokens() []string
4950
PatchesDir() string
5051
}

‎cmd/kubeadm/app/cmd/phases/init/data_test.go‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -31,22 +31,23 @@ type testInitData struct{}
3131
// testInitData must satisfy InitData.
3232
var _ InitData = &testInitData{}
3333

34-
func (t *testInitData) UploadCerts() bool { return false }
35-
func (t *testInitData) CertificateKey() string { return "" }
36-
func (t *testInitData) SetCertificateKey(key string) {}
37-
func (t *testInitData) SkipCertificateKeyPrint() bool { return false }
38-
func (t *testInitData) Cfg() *kubeadmapi.InitConfiguration { return nil }
39-
func (t *testInitData) DryRun() bool { return false }
40-
func (t *testInitData) SkipTokenPrint() bool { return false }
41-
func (t *testInitData) IgnorePreflightErrors() sets.Set[string] { return nil }
42-
func (t *testInitData) CertificateWriteDir() string { return "" }
43-
func (t *testInitData) CertificateDir() string { return "" }
44-
func (t *testInitData) KubeConfigDir() string { return "" }
45-
func (t *testInitData) KubeConfigPath() string { return "" }
46-
func (t *testInitData) ManifestDir() string { return "" }
47-
func (t *testInitData) KubeletDir() string { return "" }
48-
func (t *testInitData) ExternalCA() bool { return false }
49-
func (t *testInitData) OutputWriter() io.Writer { return nil }
50-
func (t *testInitData) Client() (clientset.Interface, error) { return nil, nil }
51-
func (t *testInitData) Tokens() []string { return nil }
52-
func (t *testInitData) PatchesDir() string { return "" }
34+
func (t *testInitData) UploadCerts() bool { return false }
35+
func (t *testInitData) CertificateKey() string { return "" }
36+
func (t *testInitData) SetCertificateKey(key string) {}
37+
func (t *testInitData) SkipCertificateKeyPrint() bool { return false }
38+
func (t *testInitData) Cfg() *kubeadmapi.InitConfiguration { return nil }
39+
func (t *testInitData) DryRun() bool { return false }
40+
func (t *testInitData) SkipTokenPrint() bool { return false }
41+
func (t *testInitData) IgnorePreflightErrors() sets.Set[string] { return nil }
42+
func (t *testInitData) CertificateWriteDir() string { return "" }
43+
func (t *testInitData) CertificateDir() string { return "" }
44+
func (t *testInitData) KubeConfigDir() string { return "" }
45+
func (t *testInitData) KubeConfigPath() string { return "" }
46+
func (t *testInitData) ManifestDir() string { return "" }
47+
func (t *testInitData) KubeletDir() string { return "" }
48+
func (t *testInitData) ExternalCA() bool { return false }
49+
func (t *testInitData) OutputWriter() io.Writer { return nil }
50+
func (t *testInitData) Client() (clientset.Interface, error) { return nil, nil }
51+
func (t *testInitData) ClientWithoutBootstrap() (clientset.Interface, error) { return nil, nil }
52+
func (t *testInitData) Tokens() []string { return nil }
53+
func (t *testInitData) PatchesDir() string { return "" }

‎cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,10 @@ func runWaitControlPlanePhase(c workflow.RunData) error {
8282
// waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled
8383
klog.V(1).Infoln("[wait-control-plane] Waiting for the API server to be healthy")
8484

85-
client, err := data.Client()
85+
// WaitForAPI uses the /healthz endpoint, thus a client without permissions works fine
86+
client, err := data.ClientWithoutBootstrap()
8687
if err != nil {
87-
return errors.Wrap(err, "cannot obtain client")
88+
return errors.Wrap(err, "cannot obtain client without bootstrap")
8889
}
8990

9091
timeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration

0 commit comments

Comments
 (0)