Skip to content

Commit 595482d

Browse files
SataQiuSataQiu
committed
kubeadm: implement 'kubeadm upgrade apply phase'
Signed-off-by: SataQiu <[email protected]>
1 parent 9f01cd7 commit 595482d

File tree

13 files changed

+1105
-238
lines changed

13 files changed

+1105
-238
lines changed

‎cmd/kubeadm/app/cmd/phases/upgrade/apply/addons.go‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package apply implements phases of 'kubeadm upgrade apply'.
18+
package apply
19+
20+
import (
21+
"context"
22+
"fmt"
23+
"io"
24+
25+
"github.com/pkg/errors"
26+
27+
apierrors "k8s.io/apimachinery/pkg/api/errors"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
clientset "k8s.io/client-go/kubernetes"
30+
"k8s.io/klog/v2"
31+
32+
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
33+
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
34+
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
35+
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
36+
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
37+
dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
38+
proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
39+
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
40+
)
41+
42+
// NewAddonPhase returns the addon Cobra command
43+
func NewAddonPhase() workflow.Phase {
44+
return workflow.Phase{
45+
Name: "addon",
46+
Short: "Install required addons for passing conformance tests",
47+
Long: cmdutil.MacroCommandLongDescription,
48+
Phases: []workflow.Phase{
49+
{
50+
Name: "all",
51+
Short: "Install all the addons",
52+
InheritFlags: getAddonPhaseFlags("all"),
53+
RunAllSiblings: true,
54+
},
55+
{
56+
Name: "coredns",
57+
Short: "Install the CoreDNS addon to a Kubernetes cluster",
58+
InheritFlags: getAddonPhaseFlags("coredns"),
59+
Run: runCoreDNSAddon,
60+
},
61+
{
62+
Name: "kube-proxy",
63+
Short: "Install the kube-proxy addon to a Kubernetes cluster",
64+
InheritFlags: getAddonPhaseFlags("kube-proxy"),
65+
Run: runKubeProxyAddon,
66+
},
67+
},
68+
}
69+
}
70+
71+
func shouldUpgradeAddons(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) (bool, error) {
72+
unupgradedControlPlanes, err := upgrade.UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
73+
if err != nil {
74+
return false, errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
75+
}
76+
if len(unupgradedControlPlanes) > 0 {
77+
fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
78+
return false, nil
79+
}
80+
81+
return true, nil
82+
}
83+
84+
func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, string, io.Writer, bool, error) {
85+
data, ok := c.(Data)
86+
if !ok {
87+
return nil, nil, "", nil, false, errors.New("addon phase invoked with an invalid data struct")
88+
}
89+
return data.InitCfg(), data.Client(), data.PatchesDir(), data.OutputWriter(), data.DryRun(), nil
90+
}
91+
92+
// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster
93+
func runCoreDNSAddon(c workflow.RunData) error {
94+
cfg, client, patchesDir, out, dryRun, err := getInitData(c)
95+
if err != nil {
96+
return err
97+
}
98+
99+
shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
100+
if err != nil {
101+
return err
102+
}
103+
if !shouldUpgradeAddons {
104+
return nil
105+
}
106+
107+
// If the coredns ConfigMap is missing, show a warning and assume that the
108+
// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
109+
//
110+
// TODO: remove this once "kubeadm upgrade apply" phases are supported:
111+
// https://github.com/kubernetes/kubeadm/issues/1318
112+
if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
113+
context.TODO(),
114+
kubeadmconstants.CoreDNSConfigMap,
115+
metav1.GetOptions{},
116+
); err != nil && apierrors.IsNotFound(err) {
117+
klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+
118+
"Assuming that a DNS server was not deployed for this cluster. "+
119+
"Note that once 'kubeadm upgrade apply' supports phases you "+
120+
"will have to skip the DNS upgrade manually",
121+
kubeadmconstants.CoreDNSConfigMap,
122+
metav1.NamespaceSystem)
123+
return nil
124+
}
125+
126+
// Upgrade CoreDNS
127+
if err := dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client, patchesDir, out, dryRun); err != nil {
128+
return err
129+
}
130+
131+
return nil
132+
}
133+
134+
// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster
135+
func runKubeProxyAddon(c workflow.RunData) error {
136+
cfg, client, _, out, dryRun, err := getInitData(c)
137+
if err != nil {
138+
return err
139+
}
140+
141+
shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
142+
if err != nil {
143+
return err
144+
}
145+
if !shouldUpgradeAddons {
146+
return nil
147+
}
148+
149+
// If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy
150+
// was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
151+
//
152+
// TODO: remove this once "kubeadm upgrade apply" phases are supported:
153+
// https://github.com/kubernetes/kubeadm/issues/1318
154+
if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
155+
context.TODO(),
156+
kubeadmconstants.KubeProxyConfigMap,
157+
metav1.GetOptions{},
158+
); err != nil && apierrors.IsNotFound(err) {
159+
klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+
160+
"Assuming that kube-proxy was not deployed for this cluster. "+
161+
"Note that once 'kubeadm upgrade apply' supports phases you "+
162+
"will have to skip the kube-proxy upgrade manually",
163+
kubeadmconstants.KubeProxyConfigMap,
164+
metav1.NamespaceSystem)
165+
return nil
166+
}
167+
168+
// Upgrade kube-proxy
169+
if err := proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, dryRun); err != nil {
170+
return err
171+
}
172+
173+
return nil
174+
}
175+
176+
func getAddonPhaseFlags(name string) []string {
177+
flags := []string{
178+
options.CfgPath,
179+
options.KubeconfigPath,
180+
options.DryRun,
181+
}
182+
if name == "all" || name == "coredns" {
183+
flags = append(flags,
184+
options.Patches,
185+
)
186+
}
187+
return flags
188+
}

‎cmd/kubeadm/app/cmd/phases/upgrade/apply/bootstraptoken.go‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package apply implements phases of 'kubeadm upgrade apply'.
18+
package apply
19+
20+
import (
21+
"fmt"
22+
23+
"github.com/pkg/errors"
24+
25+
errorsutil "k8s.io/apimachinery/pkg/util/errors"
26+
27+
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
28+
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
29+
clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
30+
nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
31+
)
32+
33+
// NewBootstrapTokenPhase returns the phase to bootstrapToken
34+
func NewBootstrapTokenPhase() workflow.Phase {
35+
return workflow.Phase{
36+
Name: "bootstrap-token",
37+
Aliases: []string{"bootstraptoken"},
38+
Short: "Generates bootstrap tokens used to join a node to a cluster",
39+
InheritFlags: []string{
40+
options.CfgPath,
41+
options.KubeconfigPath,
42+
options.DryRun,
43+
},
44+
Run: runBootstrapToken,
45+
}
46+
}
47+
48+
func runBootstrapToken(c workflow.RunData) error {
49+
data, ok := c.(Data)
50+
if !ok {
51+
return errors.New("bootstrap-token phase invoked with an invalid data struct")
52+
}
53+
54+
if data.DryRun() {
55+
fmt.Println("[dryrun] Would config cluster-info ConfigMap, RBAC Roles")
56+
return nil
57+
}
58+
59+
fmt.Println("[bootstrap-token] Configuring cluster-info ConfigMap, RBAC Roles")
60+
61+
client := data.Client()
62+
63+
var errs []error
64+
// Create RBAC rules that makes the bootstrap tokens able to get nodes
65+
if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil {
66+
errs = append(errs, err)
67+
}
68+
69+
// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
70+
if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
71+
errs = append(errs, err)
72+
}
73+
74+
// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
75+
if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
76+
errs = append(errs, err)
77+
}
78+
79+
// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
80+
if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
81+
errs = append(errs, err)
82+
}
83+
84+
// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
85+
if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil {
86+
errs = append(errs, err)
87+
}
88+
89+
return errorsutil.NewAggregate(errs)
90+
}

‎cmd/kubeadm/app/cmd/phases/upgrade/apply/controlplane.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package apply implements phases of 'kubeadm upgrade apply'.
18+
package apply
19+
20+
import (
21+
"fmt"
22+
"os"
23+
24+
"github.com/pkg/errors"
25+
26+
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
27+
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
28+
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
29+
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
30+
)
31+
32+
// NewControlPlanePhase creates a kubeadm workflow phase that implements handling of control-plane upgrade.
33+
func NewControlPlanePhase() workflow.Phase {
34+
phase := workflow.Phase{
35+
Name: "control-plane",
36+
Short: "Upgrade the control plane",
37+
Run: runControlPlane,
38+
InheritFlags: []string{
39+
options.CfgPath,
40+
options.KubeconfigPath,
41+
options.DryRun,
42+
options.CertificateRenewal,
43+
options.EtcdUpgrade,
44+
options.Patches,
45+
},
46+
}
47+
return phase
48+
}
49+
50+
func runControlPlane(c workflow.RunData) error {
51+
data, ok := c.(Data)
52+
if !ok {
53+
return errors.New("control-plane phase invoked with an invalid data struct")
54+
}
55+
56+
initCfg, upgradeCfg, client, patchesDir := data.InitCfg(), data.Cfg(), data.Client(), data.PatchesDir()
57+
58+
if data.DryRun() {
59+
fmt.Printf("[dryrun] Would upgrade your Static Pod-hosted control plane to version %q", initCfg.KubernetesVersion)
60+
return upgrade.DryRunStaticPodUpgrade(patchesDir, initCfg)
61+
}
62+
63+
fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
64+
initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration)
65+
66+
waiter := apiclient.NewKubeWaiter(client, upgradeCfg.Timeouts.UpgradeManifests.Duration, os.Stdout)
67+
if err := upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, data.EtcdUpgrade(), data.RenewCerts(), patchesDir); err != nil {
68+
return errors.Wrap(err, "couldn't complete the static pod upgrade")
69+
}
70+
71+
fmt.Println("[upgrade] The control plane instance for this node was successfully updated!")
72+
73+
return nil
74+
}

0 commit comments

Comments
 (0)