Skip to content

Commit 6f1543f

Browse files
chrischdichrischdi
committed
bootstrap: fix useExperimentalRetryJoin for kubernetes v1.31
This adds a new script for useExperimentalRetryJoin for kubernetes >= v1.31 due to the following changes: * control-plane update-status phase got removed * new phases got added for the ControlPlaneKubeletLocalMode feature gate
1 parent 0c026e9 commit 6f1543f

File tree

5 files changed

+202
-9
lines changed

5 files changed

+202
-9
lines changed

‎bootstrap/kubeadm/internal/cloudinit/cloudinit.go

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import (
2222
"fmt"
2323
"text/template"
2424

25+
"github.com/blang/semver/v4"
2526
"github.com/pkg/errors"
2627

2728
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
29+
"sigs.k8s.io/cluster-api/util/version"
2830
)
2931

3032
const (
@@ -56,6 +58,7 @@ type BaseUserData struct {
5658
KubeadmCommand string
5759
KubeadmVerbosity string
5860
SentinelFileCommand string
61+
KubernetesVersion semver.Version
5962
}
6063

6164
func (input *BaseUserData) prepare() error {
@@ -64,7 +67,7 @@ func (input *BaseUserData) prepare() error {
6467
input.KubeadmCommand = fmt.Sprintf(standardJoinCommand, input.KubeadmVerbosity)
6568
if input.UseExperimentalRetry {
6669
input.KubeadmCommand = retriableJoinScriptName
67-
joinScriptFile, err := generateBootstrapScript(input)
70+
joinScriptFile, err := generateBootstrapScript(input, input.KubernetesVersion)
6871
if err != nil {
6972
return errors.Wrap(err, "failed to generate user data for machine joining control plane")
7073
}
@@ -118,12 +121,23 @@ func generate(kind string, tpl string, data interface{}) ([]byte, error) {
118121
}
119122

120123
var (
124+
//go:embed kubeadm-bootstrap-script-pre-k8s-1-31.sh
125+
kubeadmBootstrapScriptPre1_31 string
121126
//go:embed kubeadm-bootstrap-script.sh
122127
kubeadmBootstrapScript string
128+
129+
// kubernetesVersion1_31 is the version where kubeadm removed the update-status phase
130+
// and introduced new phases with the ControlPlaneKubeletLocalMode feature gate.
131+
kubernetesVersion1_31 = semver.MustParse("1.31.0")
123132
)
124133

125-
func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) {
126-
joinScript, err := generate("JoinScript", kubeadmBootstrapScript, input)
134+
func generateBootstrapScript(input interface{}, parsedversion semver.Version) (*bootstrapv1.File, error) {
135+
bootstrapScript := kubeadmBootstrapScript
136+
if useKubeadmBootstrapScriptPre1_31(parsedversion) {
137+
bootstrapScript = kubeadmBootstrapScriptPre1_31
138+
}
139+
140+
joinScript, err := generate("JoinScript", bootstrapScript, input)
127141
if err != nil {
128142
return nil, errors.Wrap(err, "failed to bootstrap script for machine joins")
129143
}
@@ -134,3 +148,7 @@ func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) {
134148
Content: string(joinScript),
135149
}, nil
136150
}
151+
152+
func useKubeadmBootstrapScriptPre1_31(parsedversion semver.Version) bool {
153+
return version.Compare(parsedversion, kubernetesVersion1_31, version.WithoutPreReleases()) < 0
154+
}

‎bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package cloudinit
1919
import (
2020
"testing"
2121

22+
"github.com/blang/semver/v4"
2223
. "github.com/onsi/gomega"
2324
"k8s.io/utils/ptr"
2425

@@ -278,3 +279,39 @@ func TestNewJoinControlPlaneExperimentalRetry(t *testing.T) {
278279
g.Expect(out).To(ContainSubstring(f))
279280
}
280281
}
282+
283+
func Test_useKubeadmBootstrapScriptPre1_31(t *testing.T) {
284+
tests := []struct {
285+
name string
286+
parsedversion semver.Version
287+
want bool
288+
}{
289+
{
290+
name: "true for version for v1.30",
291+
parsedversion: semver.MustParse("1.30.99"),
292+
want: true,
293+
},
294+
{
295+
name: "true for version for v1.28",
296+
parsedversion: semver.MustParse("1.28.0"),
297+
want: true,
298+
},
299+
{
300+
name: "false for v1.31.0",
301+
parsedversion: semver.MustParse("1.31.0"),
302+
want: false,
303+
},
304+
{
305+
name: "false for v1.31.0-beta.0",
306+
parsedversion: semver.MustParse("1.31.0-beta.0"),
307+
want: false,
308+
},
309+
}
310+
for _, tt := range tests {
311+
t.Run(tt.name, func(t *testing.T) {
312+
if got := useKubeadmBootstrapScriptPre1_31(tt.parsedversion); got != tt.want {
313+
t.Errorf("useKubeadmBootstrapScriptPre1_31() = %v, want %v", got, tt.want)
314+
}
315+
})
316+
}
317+
}

‎bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
#!/bin/bash
2+
# Copyright 2020 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Log an error and exit.
17+
# Args:
18+
# $1 Message to log with the error
19+
# $2 The error code to return
20+
log::error_exit() {
21+
local message="${1}"
22+
local code="${2}"
23+
24+
log::error "${message}"
25+
# {{ if .ControlPlane }}
26+
log::info "Removing member from cluster status"
27+
kubeadm reset -f update-cluster-status || true
28+
log::info "Removing etcd member"
29+
kubeadm reset -f remove-etcd-member || true
30+
# {{ end }}
31+
log::info "Resetting kubeadm"
32+
kubeadm reset -f || true
33+
log::error "cluster.x-k8s.io kubeadm bootstrap script $0 exiting with status ${code}"
34+
exit "${code}"
35+
}
36+
37+
log::success_exit() {
38+
log::info "cluster.x-k8s.io kubeadm bootstrap script $0 finished"
39+
exit 0
40+
}
41+
42+
# Log an error but keep going.
43+
log::error() {
44+
local message="${1}"
45+
timestamp=$(date --iso-8601=seconds)
46+
echo "!!! [${timestamp}] ${1}" >&2
47+
shift
48+
for message; do
49+
echo " ${message}" >&2
50+
done
51+
}
52+
53+
# Print a status line. Formatted to show up in a stream of output.
54+
log::info() {
55+
timestamp=$(date --iso-8601=seconds)
56+
echo "+++ [${timestamp}] ${1}"
57+
shift
58+
for message; do
59+
echo " ${message}"
60+
done
61+
}
62+
63+
check_kubeadm_command() {
64+
local command="${1}"
65+
local code="${2}"
66+
case ${code} in
67+
"0")
68+
log::info "kubeadm reported successful execution for ${command}"
69+
;;
70+
"1")
71+
log::error "kubeadm reported failed action(s) for ${command}"
72+
;;
73+
"2")
74+
log::error "kubeadm reported preflight check error during ${command}"
75+
;;
76+
"3")
77+
log::error_exit "kubeadm reported validation error for ${command}" "${code}"
78+
;;
79+
*)
80+
log::error "kubeadm reported unknown error ${code} for ${command}"
81+
;;
82+
esac
83+
}
84+
85+
function retry-command() {
86+
n=0
87+
local kubeadm_return
88+
until [ $n -ge 5 ]; do
89+
log::info "running '$*'"
90+
# shellcheck disable=SC1083
91+
"$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}}
92+
kubeadm_return=$?
93+
check_kubeadm_command "'$*'" "${kubeadm_return}"
94+
if [ ${kubeadm_return} -eq 0 ]; then
95+
break
96+
fi
97+
# We allow preflight errors to pass
98+
if [ ${kubeadm_return} -eq 2 ]; then
99+
break
100+
fi
101+
n=$((n + 1))
102+
sleep 15
103+
done
104+
if [ ${kubeadm_return} -ne 0 ]; then
105+
log::error_exit "too many errors, exiting" "${kubeadm_return}"
106+
fi
107+
}
108+
109+
# {{ if .ControlPlane }}
110+
function try-or-die-command() {
111+
local kubeadm_return
112+
log::info "running '$*'"
113+
# shellcheck disable=SC1083
114+
"$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}}
115+
kubeadm_return=$?
116+
check_kubeadm_command "'$*'" "${kubeadm_return}"
117+
if [ ${kubeadm_return} -ne 0 ]; then
118+
log::error_exit "fatal error, exiting" "${kubeadm_return}"
119+
fi
120+
}
121+
# {{ end }}
122+
123+
retry-command kubeadm join phase preflight --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests
124+
# {{ if .ControlPlane }}
125+
retry-command kubeadm join phase control-plane-prepare download-certs
126+
retry-command kubeadm join phase control-plane-prepare certs
127+
retry-command kubeadm join phase control-plane-prepare kubeconfig
128+
retry-command kubeadm join phase control-plane-prepare control-plane
129+
# {{ end }}
130+
retry-command kubeadm join phase kubelet-start
131+
# {{ if .ControlPlane }}
132+
try-or-die-command kubeadm join phase control-plane-join etcd
133+
retry-command kubeadm join phase control-plane-join update-status
134+
retry-command kubeadm join phase control-plane-join mark-control-plane
135+
# {{ end }}
136+
137+
log::success_exit

‎bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright 2020 The Kubernetes Authors.
2+
# Copyright 2024 The Kubernetes Authors.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -128,10 +128,8 @@ retry-command kubeadm join phase control-plane-prepare kubeconfig
128128
retry-command kubeadm join phase control-plane-prepare control-plane
129129
# {{ end }}
130130
retry-command kubeadm join phase kubelet-start
131-
# {{ if .ControlPlane }}
132-
try-or-die-command kubeadm join phase control-plane-join etcd
133-
retry-command kubeadm join phase control-plane-join update-status
134-
retry-command kubeadm join phase control-plane-join mark-control-plane
135-
# {{ end }}
131+
132+
# Run kubeadm join and skip all already executed phases.
133+
try-or-die-command kubeadm join --skip-phases preflight,control-plane-prepare,kubelet-start
136134

137135
log::success_exit

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
535535
Mounts: scope.Config.Spec.Mounts,
536536
DiskSetup: scope.Config.Spec.DiskSetup,
537537
KubeadmVerbosity: verbosityFlag,
538+
KubernetesVersion: parsedVersion,
538539
},
539540
InitConfiguration: initdata,
540541
ClusterConfiguration: clusterdata,
@@ -656,6 +657,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
656657
DiskSetup: scope.Config.Spec.DiskSetup,
657658
KubeadmVerbosity: verbosityFlag,
658659
UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin,
660+
KubernetesVersion: parsedVersion,
659661
},
660662
JoinConfiguration: joinData,
661663
}
@@ -774,6 +776,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
774776
DiskSetup: scope.Config.Spec.DiskSetup,
775777
KubeadmVerbosity: verbosityFlag,
776778
UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin,
779+
KubernetesVersion: parsedVersion,
777780
},
778781
}
779782

0 commit comments

Comments
 (0)