Skip to content

Commit a758ae0

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #17539 from hakman/etcd-manager-image-volumes
etcd-manager: Use image volumes to mount etcd images
2 parents 5e8e232 + da4af99 commit a758ae0

15 files changed

+273
-424
lines changed

‎pkg/apis/kops/cluster.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package kops
1919
import (
2020
"fmt"
2121

22+
"github.com/blang/semver/v4"
2223
corev1 "k8s.io/api/core/v1"
2324
"k8s.io/apimachinery/pkg/api/resource"
2425
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -947,6 +948,29 @@ func (c *Cluster) InstallCNIAssets() bool {
947948
c.Spec.Networking.Cilium == nil
948949
}
949950

951+
func (c *Cluster) HasImageVolumesSupport() bool {
952+
// Image Volumes was added to Kubernetes v1.31
953+
// https://kubernetes.io/blog/2024/08/16/kubernetes-1-31-image-volume-source/
954+
// Image Volumes graduated to beta in Kubernetes v1.33
955+
// https://kubernetes.io/blog/2025/04/29/kubernetes-v1-33-image-volume-beta/
956+
if c.IsKubernetesLT("1.33.0") {
957+
return false
958+
}
959+
if c.Spec.Containerd == nil || c.Spec.Containerd.Version == nil {
960+
return false
961+
}
962+
sv, err := semver.ParseTolerant(*c.Spec.Containerd.Version)
963+
if err != nil {
964+
return false
965+
}
966+
// Image Volumes was released in Containerd v2.1.0
967+
// https://github.com/containerd/containerd/releases/tag/v2.1.0
968+
if sv.LT(semver.MustParse("2.1.0")) {
969+
return false
970+
}
971+
return true
972+
}
973+
950974
func (c *Cluster) APIInternalName() string {
951975
return "api.internal." + c.ObjectMeta.Name
952976
}

‎pkg/model/components/etcdmanager/model.go‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,22 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
236236
}
237237
}
238238

239-
{
239+
if b.Cluster.HasImageVolumesSupport() {
240+
for _, etcdVersion := range etcdSupportedVersions() {
241+
if etcdVersion.SymlinkToVersion == "" {
242+
volume := v1.Volume{
243+
Name: "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-"),
244+
VolumeSource: v1.VolumeSource{
245+
Image: &v1.ImageVolumeSource{
246+
Reference: b.AssetBuilder.RemapImage(etcdVersion.Image),
247+
PullPolicy: v1.PullIfNotPresent,
248+
},
249+
},
250+
}
251+
pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
252+
}
253+
}
254+
} else {
240255
utilMounts := []v1.VolumeMount{
241256
{
242257
MountPath: "/opt",
@@ -331,6 +346,20 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
331346

332347
// Remap image via AssetBuilder
333348
container.Image = b.AssetBuilder.RemapImage(container.Image)
349+
350+
if b.Cluster.HasImageVolumesSupport() {
351+
for _, etcdVersion := range etcdSupportedVersions() {
352+
volumeMount := v1.VolumeMount{
353+
MountPath: "/opt/etcd-v" + etcdVersion.Version,
354+
}
355+
if etcdVersion.SymlinkToVersion == "" {
356+
volumeMount.Name = "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-")
357+
} else {
358+
volumeMount.Name = "etcd-v" + strings.ReplaceAll(etcdVersion.SymlinkToVersion, ".", "-")
359+
}
360+
container.VolumeMounts = append(container.VolumeMounts, volumeMount)
361+
}
362+
}
334363
}
335364

336365
var clientHost string

‎pkg/model/components/etcdmanager/options.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ var _ loader.ClusterOptionsBuilder = &EtcdManagerOptionsBuilder{}
4040
func (b *EtcdManagerOptionsBuilder) BuildOptions(o *kops.Cluster) error {
4141
clusterSpec := &o.Spec
4242

43+
// Image Volumes will become GA in Kubernetes 1.35
44+
// https://github.com/kubernetes/enhancements/pull/5450
45+
if b.ControlPlaneKubernetesVersion().IsLT("1.35.0") && o.HasImageVolumesSupport() {
46+
if clusterSpec.ControlPlaneKubelet == nil {
47+
clusterSpec.ControlPlaneKubelet = &kops.KubeletConfigSpec{}
48+
}
49+
if clusterSpec.ControlPlaneKubelet.FeatureGates == nil {
50+
clusterSpec.ControlPlaneKubelet.FeatureGates = make(map[string]string)
51+
}
52+
if _, found := clusterSpec.ControlPlaneKubelet.FeatureGates["ImageVolume"]; !found {
53+
clusterSpec.ControlPlaneKubelet.FeatureGates["ImageVolume"] = "true"
54+
}
55+
}
56+
4357
for i := range clusterSpec.EtcdClusters {
4458
etcdCluster := &clusterSpec.EtcdClusters[i]
4559
if etcdCluster.Backups == nil {

‎pkg/model/components/etcdmanager/tests/minimal/cluster.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ spec:
2828
provider: Manager
2929
backups:
3030
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
31-
kubernetesVersion: v1.21.0
31+
containerd:
32+
version: 2.1.4
33+
kubernetesVersion: v1.33.0
3234
masterPublicName: api.minimal.example.com
3335
networkCIDR: 172.20.0.0/16
3436
networking:

‎pkg/model/components/etcdmanager/tests/minimal/tasks.yaml‎

Lines changed: 64 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -102,80 +102,34 @@ Contents: |
102102
name: pki
103103
- mountPath: /opt
104104
name: opt
105+
- mountPath: /opt/etcd-v3.4.13
106+
name: etcd-v3-4-13
107+
- mountPath: /opt/etcd-v3.4.3
108+
name: etcd-v3-4-13
109+
- mountPath: /opt/etcd-v3.5.0
110+
name: etcd-v3-5-21
111+
- mountPath: /opt/etcd-v3.5.1
112+
name: etcd-v3-5-21
113+
- mountPath: /opt/etcd-v3.5.13
114+
name: etcd-v3-5-21
115+
- mountPath: /opt/etcd-v3.5.17
116+
name: etcd-v3-5-21
117+
- mountPath: /opt/etcd-v3.5.21
118+
name: etcd-v3-5-21
119+
- mountPath: /opt/etcd-v3.5.3
120+
name: etcd-v3-5-21
121+
- mountPath: /opt/etcd-v3.5.4
122+
name: etcd-v3-5-21
123+
- mountPath: /opt/etcd-v3.5.6
124+
name: etcd-v3-5-21
125+
- mountPath: /opt/etcd-v3.5.7
126+
name: etcd-v3-5-21
127+
- mountPath: /opt/etcd-v3.5.9
128+
name: etcd-v3-5-21
105129
- mountPath: /var/log/etcd.log
106130
name: varlogetcd
107131
hostNetwork: true
108132
hostPID: true
109-
initContainers:
110-
- args:
111-
- --target-dir=/opt/kops-utils/
112-
- --src=/ko-app/kops-utils-cp
113-
command:
114-
- /ko-app/kops-utils-cp
115-
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
116-
name: kops-utils-cp
117-
resources: {}
118-
volumeMounts:
119-
- mountPath: /opt
120-
name: opt
121-
- args:
122-
- --target-dir=/opt/etcd-v3.4.13
123-
- --src=/usr/local/bin/etcd
124-
- --src=/usr/local/bin/etcdctl
125-
command:
126-
- /opt/kops-utils/kops-utils-cp
127-
image: registry.k8s.io/etcd:v3.4.13
128-
name: init-etcd-3-4-13
129-
resources: {}
130-
volumeMounts:
131-
- mountPath: /opt
132-
name: opt
133-
- args:
134-
- --target-dir=/opt/etcd-v3.5.21
135-
- --src=/usr/local/bin/etcd
136-
- --src=/usr/local/bin/etcdctl
137-
command:
138-
- /opt/kops-utils/kops-utils-cp
139-
image: registry.k8s.io/etcd:v3.5.21
140-
name: init-etcd-3-5-21
141-
resources: {}
142-
volumeMounts:
143-
- mountPath: /opt
144-
name: opt
145-
- args:
146-
- --symlink
147-
- --target-dir=/opt/etcd-v3.4.3
148-
- --src=/opt/etcd-v3.4.13/etcd
149-
- --src=/opt/etcd-v3.4.13/etcdctl
150-
command:
151-
- /opt/kops-utils/kops-utils-cp
152-
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
153-
name: init-etcd-symlinks-3-4-13
154-
resources: {}
155-
volumeMounts:
156-
- mountPath: /opt
157-
name: opt
158-
- args:
159-
- --symlink
160-
- --target-dir=/opt/etcd-v3.5.0
161-
- --target-dir=/opt/etcd-v3.5.1
162-
- --target-dir=/opt/etcd-v3.5.13
163-
- --target-dir=/opt/etcd-v3.5.17
164-
- --target-dir=/opt/etcd-v3.5.3
165-
- --target-dir=/opt/etcd-v3.5.4
166-
- --target-dir=/opt/etcd-v3.5.6
167-
- --target-dir=/opt/etcd-v3.5.7
168-
- --target-dir=/opt/etcd-v3.5.9
169-
- --src=/opt/etcd-v3.5.21/etcd
170-
- --src=/opt/etcd-v3.5.21/etcdctl
171-
command:
172-
- /opt/kops-utils/kops-utils-cp
173-
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
174-
name: init-etcd-symlinks-3-5-21
175-
resources: {}
176-
volumeMounts:
177-
- mountPath: /opt
178-
name: opt
179133
priorityClassName: system-cluster-critical
180134
tolerations:
181135
- key: CriticalAddonsOnly
@@ -195,6 +149,14 @@ Contents: |
195149
name: pki
196150
- emptyDir: {}
197151
name: opt
152+
- image:
153+
pullPolicy: IfNotPresent
154+
reference: registry.k8s.io/etcd:v3.4.13
155+
name: etcd-v3-4-13
156+
- image:
157+
pullPolicy: IfNotPresent
158+
reference: registry.k8s.io/etcd:v3.5.21
159+
name: etcd-v3-5-21
198160
- hostPath:
199161
path: /var/log/etcd-events.log
200162
type: FileOrCreate
@@ -244,80 +206,34 @@ Contents: |
244206
name: pki
245207
- mountPath: /opt
246208
name: opt
209+
- mountPath: /opt/etcd-v3.4.13
210+
name: etcd-v3-4-13
211+
- mountPath: /opt/etcd-v3.4.3
212+
name: etcd-v3-4-13
213+
- mountPath: /opt/etcd-v3.5.0
214+
name: etcd-v3-5-21
215+
- mountPath: /opt/etcd-v3.5.1
216+
name: etcd-v3-5-21
217+
- mountPath: /opt/etcd-v3.5.13
218+
name: etcd-v3-5-21
219+
- mountPath: /opt/etcd-v3.5.17
220+
name: etcd-v3-5-21
221+
- mountPath: /opt/etcd-v3.5.21
222+
name: etcd-v3-5-21
223+
- mountPath: /opt/etcd-v3.5.3
224+
name: etcd-v3-5-21
225+
- mountPath: /opt/etcd-v3.5.4
226+
name: etcd-v3-5-21
227+
- mountPath: /opt/etcd-v3.5.6
228+
name: etcd-v3-5-21
229+
- mountPath: /opt/etcd-v3.5.7
230+
name: etcd-v3-5-21
231+
- mountPath: /opt/etcd-v3.5.9
232+
name: etcd-v3-5-21
247233
- mountPath: /var/log/etcd.log
248234
name: varlogetcd
249235
hostNetwork: true
250236
hostPID: true
251-
initContainers:
252-
- args:
253-
- --target-dir=/opt/kops-utils/
254-
- --src=/ko-app/kops-utils-cp
255-
command:
256-
- /ko-app/kops-utils-cp
257-
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
258-
name: kops-utils-cp
259-
resources: {}
260-
volumeMounts:
261-
- mountPath: /opt
262-
name: opt
263-
- args:
264-
- --target-dir=/opt/etcd-v3.4.13
265-
- --src=/usr/local/bin/etcd
266-
- --src=/usr/local/bin/etcdctl
267-
command:
268-
- /opt/kops-utils/kops-utils-cp
269-
image: registry.k8s.io/etcd:v3.4.13
270-
name: init-etcd-3-4-13
271-
resources: {}
272-
volumeMounts:
273-
- mountPath: /opt
274-
name: opt
275-
- args:
276-
- --target-dir=/opt/etcd-v3.5.21
277-
- --src=/usr/local/bin/etcd
278-
- --src=/usr/local/bin/etcdctl
279-
command:
280-
- /opt/kops-utils/kops-utils-cp
281-
image: registry.k8s.io/etcd:v3.5.21
282-
name: init-etcd-3-5-21
283-
resources: {}
284-
volumeMounts:
285-
- mountPath: /opt
286-
name: opt
287-
- args:
288-
- --symlink
289-
- --target-dir=/opt/etcd-v3.4.3
290-
- --src=/opt/etcd-v3.4.13/etcd
291-
- --src=/opt/etcd-v3.4.13/etcdctl
292-
command:
293-
- /opt/kops-utils/kops-utils-cp
294-
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
295-
name: init-etcd-symlinks-3-4-13
296-
resources: {}
297-
volumeMounts:
298-
- mountPath: /opt
299-
name: opt
300-
- args:
301-
- --symlink
302-
- --target-dir=/opt/etcd-v3.5.0
303-
- --target-dir=/opt/etcd-v3.5.1
304-
- --target-dir=/opt/etcd-v3.5.13
305-
- --target-dir=/opt/etcd-v3.5.17
306-
- --target-dir=/opt/etcd-v3.5.3
307-
- --target-dir=/opt/etcd-v3.5.4
308-
- --target-dir=/opt/etcd-v3.5.6
309-
- --target-dir=/opt/etcd-v3.5.7
310-
- --target-dir=/opt/etcd-v3.5.9
311-
- --src=/opt/etcd-v3.5.21/etcd
312-
- --src=/opt/etcd-v3.5.21/etcdctl
313-
command:
314-
- /opt/kops-utils/kops-utils-cp
315-
image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1
316-
name: init-etcd-symlinks-3-5-21
317-
resources: {}
318-
volumeMounts:
319-
- mountPath: /opt
320-
name: opt
321237
priorityClassName: system-cluster-critical
322238
tolerations:
323239
- key: CriticalAddonsOnly
@@ -337,6 +253,14 @@ Contents: |
337253
name: pki
338254
- emptyDir: {}
339255
name: opt
256+
- image:
257+
pullPolicy: IfNotPresent
258+
reference: registry.k8s.io/etcd:v3.4.13
259+
name: etcd-v3-4-13
260+
- image:
261+
pullPolicy: IfNotPresent
262+
reference: registry.k8s.io/etcd:v3.5.21
263+
name: etcd-v3-5-21
340264
- hostPath:
341265
path: /var/log/etcd.log
342266
type: FileOrCreate

‎tests/integration/update_cluster/minimal-1.33/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ ClusterName: minimal.example.com
130130
ConfigBase: memfs://tests/minimal.example.com
131131
InstanceGroupName: master-us-test-1a
132132
InstanceGroupRole: ControlPlane
133-
NodeupConfigHash: xbmJGssuIxsJRK9XhaKClsBg2o8f0AgSGPWoqHTTX7s=
133+
NodeupConfigHash: PtkgcAnStiVeR9dBVfWvjbyz7gMCuGX83Y49gsjcrqU=
134134
135135
__EOF_KUBE_ENV
136136

‎tests/integration/update_cluster/minimal-1.33/data/aws_s3_object_cluster-completed.spec_content‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,8 @@ spec:
170170
clusterDomain: cluster.local
171171
enableDebuggingHandlers: true
172172
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
173+
featureGates:
174+
ImageVolume: "true"
173175
kubeconfigPath: /var/lib/kubelet/kubeconfig
174176
logLevel: 2
175177
podInfraContainerImage: registry.k8s.io/pause:3.9

0 commit comments

Comments
 (0)