Merge pull request #8880 from fabriziopandini/add-kind-mapper

🐛 Add kind mapper

Author: k8s-ci-robot

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml

@@ -220,34 +220,6 @@ spec:
             valueFrom:
               template: |
                 kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
-  - name: replaceImage-v1.23.17-machineDeployment
-    description: "Sets the container image for MD DockerMachineTemplates using Kubernetes v1.23.17."
-    enabledIf: '{{ semverCompare "v1.23.17" .builtin.machineDeployment.version }}'
-    definitions:
-      - selector:
-          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-          kind: DockerMachineTemplate
-          matchResources:
-            machineDeploymentClass:
-              names:
-                - default-worker
-        jsonPatches:
-          - op: add
-            path: "/spec/template/spec/customImage"
-            value: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
-  - name: replaceImage-v1.23.17-controlPlane
-    description: "Sets the container image for CP DockerMachineTemplates using Kubernetes v1.23.17."
-    enabledIf: '{{ semverCompare "v1.23.17" .builtin.controlPlane.version }}'
-    definitions:
-      - selector:
-          apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-          kind: DockerMachineTemplate
-          matchResources:
-            controlPlane: true
-        jsonPatches:
-          - op: add
-            path: "/spec/template/spec/customImage"
-            value: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
   - name: preloadImages
     description: |
       Sets the container images to preload to the node that is used for running dockerMachines.
@@ -535,4 +507,3 @@ spec:
           kubeletExtraArgs:
             eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
             fail-swap-on: "false"
-

test/extension/handlers/topologymutation/handler.go

@@ -24,7 +24,6 @@ package topologymutation
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/blang/semver"
 	"github.com/pkg/errors"
@@ -40,6 +39,7 @@ import (
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	"sigs.k8s.io/cluster-api/exp/runtime/topologymutation"
 	infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/infrastructure/kind"
 	"sigs.k8s.io/cluster-api/util/version"
 )
 
@@ -257,7 +257,9 @@ func patchKubeadmConfigTemplate(ctx context.Context, k *bootstrapv1.KubeadmConfi
 
 // patchDockerMachineTemplate patches the DockerMachineTemplate.
 // It sets the CustomImage to an image for the version in use by the controlPlane or by the MachineDeployment
-// the DockerMachineTemplate belongs to. This patch is required to pick up the kind image with the required Kubernetes version.
+// the DockerMachineTemplate belongs to.
+// NOTE: this patch is not required anymore after the introduction of the kind mapper in kind, however we keep it
+// as example of version aware patches.
 func patchDockerMachineTemplate(ctx context.Context, dockerMachineTemplate *infrav1.DockerMachineTemplate, templateVariables map[string]apiextensionsv1.JSON) error {
 	log := ctrl.LoggerFrom(ctx)
 
@@ -270,13 +272,14 @@ func patchDockerMachineTemplate(ctx context.Context, dockerMachineTemplate *infr
 		return errors.Wrap(err, "could not set customImage to control plane dockerMachineTemplate")
 	}
 	if found {
-		_, err := version.ParseMajorMinorPatchTolerant(cpVersion)
+		semVer, err := version.ParseMajorMinorPatchTolerant(cpVersion)
 		if err != nil {
 			return errors.Wrap(err, "could not parse control plane version")
 		}
-		customImage := fmt.Sprintf("kindest/node:%s", strings.ReplaceAll(cpVersion, "+", "_"))
-		log.Info(fmt.Sprintf("Setting MachineDeployment custom image to %q", customImage))
-		dockerMachineTemplate.Spec.Template.Spec.CustomImage = customImage
+		kindMapping := kind.GetMapping(semVer, "")
+
+		log.Info(fmt.Sprintf("Setting MachineDeployment custom image to %q", kindMapping.Image))
+		dockerMachineTemplate.Spec.Template.Spec.CustomImage = kindMapping.Image
 		// return early if we have successfully patched a control plane dockerMachineTemplate
 		return nil
 	}
@@ -290,13 +293,14 @@ func patchDockerMachineTemplate(ctx context.Context, dockerMachineTemplate *infr
 		return errors.Wrap(err, "could not set customImage to MachineDeployment DockerMachineTemplate")
 	}
 	if found {
-		_, err := version.ParseMajorMinorPatchTolerant(mdVersion)
+		semVer, err := version.ParseMajorMinorPatchTolerant(mdVersion)
 		if err != nil {
 			return errors.Wrap(err, "could not parse MachineDeployment version")
 		}
-		customImage := fmt.Sprintf("kindest/node:%s", strings.ReplaceAll(mdVersion, "+", "_"))
-		log.Info(fmt.Sprintf("Setting MachineDeployment customImage to %q", customImage))
-		dockerMachineTemplate.Spec.Template.Spec.CustomImage = customImage
+		kindMapping := kind.GetMapping(semVer, "")
+
+		log.Info(fmt.Sprintf("Setting MachineDeployment customImage to %q", kindMapping.Image))
+		dockerMachineTemplate.Spec.Template.Spec.CustomImage = kindMapping.Image
 		return nil
 	}
 

test/infrastructure/container/docker.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/utils/pointer"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/infrastructure/kind"
 )
 
 const (
@@ -390,6 +391,8 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine
 		restartMaximumRetryCount = 1
 	}
 
+	// TODO: check if we can simplify the following code for the CAPD load balancer, which now always has runConfig.KindMode == kind.ModeNone
+
 	hostConfig := dockercontainer.HostConfig{
 		// Running containers in a container requires privileges.
 		// NOTE: we could try to replicate this with --cap-add, and use less
@@ -406,6 +409,11 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine
 	}
 	networkConfig := network.NetworkingConfig{}
 
+	// NOTE: starting from Kind 0.20 kind requires CgroupnsMode to be set to private.
+	if runConfig.KindMode != kind.ModeNone && runConfig.KindMode != kind.Mode0_19 {
+		hostConfig.CgroupnsMode = "private"
+	}
+
 	if runConfig.IPFamily == clusterv1.IPv6IPFamily || runConfig.IPFamily == clusterv1.DualStackIPFamily {
 		hostConfig.Sysctls = map[string]string{
 			"net.ipv6.conf.all.disable_ipv6": "0",

test/infrastructure/container/interface.go

@@ -22,6 +22,7 @@ import (
 	"io"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/infrastructure/kind"
 )
 
 // providerKey is the key type for accessing the runtime provider in passed contexts.
@@ -98,6 +99,8 @@ type RunContainerInput struct {
 	// RestartPolicy to use for the container.
 	// If not set, defaults to "unless-stopped".
 	RestartPolicy string
+	// Defines how the kindest/node image must be started.
+	KindMode kind.Mode
 }
 
 // ExecContainerInput contains values for running exec on a container.

test/infrastructure/docker/exp/internal/docker/nodepool.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/blang/semver"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
@@ -37,8 +38,8 @@ import (
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	infraexpv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker"
+	"sigs.k8s.io/cluster-api/test/infrastructure/kind"
 	"sigs.k8s.io/cluster-api/util"
-	"sigs.k8s.io/cluster-api/util/container"
 )
 
 const (
@@ -168,15 +169,18 @@ func (np *NodePool) Delete(ctx context.Context) error {
 }
 
 func (np *NodePool) isMachineMatchingInfrastructureSpec(machine *docker.Machine) bool {
-	return imageVersion(machine) == container.SemverToOCIImageTag(*np.machinePool.Spec.Template.Spec.Version)
-}
+	// NOTE: With the current implementation we are checking if the machine is using a kindest/node image for the expected version,
+	// but not checking if the machine has the expected extra.mounts or pre.loaded images.
+
+	semVer, err := semver.Parse(strings.TrimPrefix(*np.machinePool.Spec.Template.Spec.Version, "v"))
+	if err != nil {
+		// TODO: consider if to return an error
+		panic(errors.Wrap(err, "failed to parse DockerMachine version").Error())
+	}
+
+	kindMapping := kind.GetMapping(semVer, np.dockerMachinePool.Spec.Template.CustomImage)
 
-// ImageVersion returns the version of the image used or nil if not specified
-// NOTE: Image version might be different from the Kubernetes version, because some characters
-// allowed by semver (e.g. +) can't be used for image tags, so they are replaced with "_".
-func imageVersion(m *docker.Machine) string {
-	containerImage := m.ContainerImage()
-	return containerImage[strings.LastIndex(containerImage, ":")+1:]
+	return machine.ContainerImage() == kindMapping.Image
 }
 
 // machinesMatchingInfrastructureSpec returns all of the docker.Machines which match the machine pool / docker machine pool spec.

test/infrastructure/docker/internal/docker/machine.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/blang/semver"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -45,22 +46,17 @@ import (
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/cloudinit"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/ignition"
-	clusterapicontainer "sigs.k8s.io/cluster-api/util/container"
+	"sigs.k8s.io/cluster-api/test/infrastructure/kind"
 	"sigs.k8s.io/cluster-api/util/patch"
 )
 
-const (
-	defaultImageName = "kindest/node"
-	defaultImageTag  = "v1.27.1"
-)
-
 var (
 	cloudProviderTaint = corev1.Taint{Key: "node.cloudprovider.kubernetes.io/uninitialized", Effect: corev1.TaintEffectNoSchedule}
 )
 
 type nodeCreator interface {
-	CreateControlPlaneNode(ctx context.Context, name, image, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (node *types.Node, err error)
-	CreateWorkerNode(ctx context.Context, name, image, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (node *types.Node, err error)
+	CreateControlPlaneNode(ctx context.Context, name, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (node *types.Node, err error)
+	CreateWorkerNode(ctx context.Context, name, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (node *types.Node, err error)
 }
 
 // Machine implement a service for managing the docker containers hosting a kubernetes nodes.
@@ -212,40 +208,49 @@ func (m *Machine) Create(ctx context.Context, image string, role string, version
 	if m.container == nil {
 		var err error
 
-		machineImage := m.machineImage(version)
-		if image != "" {
-			machineImage = image
+		// Get the KindMapping for the target K8s version.
+		// NOTE: The KindMapping allows to select the most recent kindest/node image available, if any, as well as
+		// provide info about the mode to be used when starting the kindest/node image itself.
+		if version == nil {
+			return errors.New("cannot create a DockerMachine for a nil version")
 		}
 
+		semVer, err := semver.Parse(strings.TrimPrefix(*version, "v"))
+		if err != nil {
+			return errors.Wrap(err, "failed to parse DockerMachine version")
+		}
+
+		kindMapping := kind.GetMapping(semVer, image)
+
 		switch role {
 		case constants.ControlPlaneNodeRoleValue:
-			log.Info(fmt.Sprintf("Creating control plane machine container with image %s", machineImage))
+			log.Info(fmt.Sprintf("Creating control plane machine container with image %s, mode %s", kindMapping.Image, kindMapping.Mode))
 			m.container, err = m.nodeCreator.CreateControlPlaneNode(
 				ctx,
 				m.ContainerName(),
-				machineImage,
 				m.cluster,
 				"127.0.0.1",
 				0,
 				kindMounts(mounts),
 				nil,
 				labels,
 				m.ipFamily,
+				kindMapping,
 			)
 			if err != nil {
 				return errors.WithStack(err)
 			}
 		case constants.WorkerNodeRoleValue:
-			log.Info(fmt.Sprintf("Creating worker machine container with image %s", machineImage))
+			log.Info(fmt.Sprintf("Creating worker machine container with image %s, mode %s", kindMapping.Image, kindMapping.Mode))
 			m.container, err = m.nodeCreator.CreateWorkerNode(
 				ctx,
 				m.ContainerName(),
-				machineImage,
 				m.cluster,
 				kindMounts(mounts),
 				nil,
 				labels,
 				m.ipFamily,
+				kindMapping,
 			)
 			if err != nil {
 				return errors.WithStack(err)
@@ -527,26 +532,6 @@ func (m *Machine) Delete(ctx context.Context) error {
 	return nil
 }
 
-// machineImage is the image of the container node with the machine.
-func (m *Machine) machineImage(version *string) string {
-	if version == nil {
-		defaultImage := fmt.Sprintf("%s:%s", defaultImageName, defaultImageTag)
-		return defaultImage
-	}
-
-	// TODO(fp) make this smarter
-	// - allows usage of custom docker repository & image names
-	// - add v only for semantic versions
-	versionString := *version
-	if !strings.HasPrefix(versionString, "v") {
-		versionString = fmt.Sprintf("v%s", versionString)
-	}
-
-	versionString = clusterapicontainer.SemverToOCIImageTag(versionString)
-
-	return fmt.Sprintf("%s:%s", defaultImageName, versionString)
-}
-
 func logContainerDebugInfo(ctx context.Context, log logr.Logger, name string) {
 	containerRuntime, err := container.RuntimeFrom(ctx)
 	if err != nil {